fix: signpost aggregate-suppressed components in a category that still has rows (#9)
This commit is contained in:
+2
-1
@@ -395,7 +395,8 @@ cog_spending <- function(govid, years, category = NULL,
|
||||
}
|
||||
suggestions <- .build_suggestions(con, govid, years, category,
|
||||
direct_leg_result,
|
||||
resolved$basis, flow_prefixes)
|
||||
resolved$basis, flow_prefixes,
|
||||
.select_long_view(view_base, resolved$basis))
|
||||
}
|
||||
|
||||
# C1(b): when expenditure_concept = "total", flag any row where the IG
|
||||
|
||||
+67
-28
@@ -1,8 +1,16 @@
|
||||
# R/suggestions.R
|
||||
# Recipe-component-driven signposting: when a basis = "harmonized" query for
|
||||
# a category comes back with a coverage gap in some requested years (the
|
||||
# result has no rows at all in that year) that a harmonization recipe would
|
||||
# actually fill for this government, surface that recipe as a suggestion.
|
||||
# Recipe-component-driven signposting. When a basis = "harmonized" query for
|
||||
# a category comes back incomplete in some requested year -- and a
|
||||
# harmonization recipe would actually fill it for this government -- surface
|
||||
# that recipe as a suggestion. "Incomplete" has two forms, and a recipe
|
||||
# qualifies on either:
|
||||
# 1. empty_year -- the result has no rows at all in that year.
|
||||
# 2. suppressed_component -- the result HAS rows, but a component code
|
||||
# carries dollars the verb's own long view structurally excludes
|
||||
# (aggregate-published, or absent from summary_categories). This is
|
||||
# uscogdata#9: Public Welfare kept returning E74/E79 rows while dropping
|
||||
# aggregate-only E67/E68, so form 1 never fired and the caller got a
|
||||
# number a third too low with no signpost at all.
|
||||
#
|
||||
# This is deliberately keyed off the recipe catalog's component codes, not
|
||||
# off harmonization_map rows: no live map row carries a non-blank
|
||||
@@ -48,11 +56,15 @@
|
||||
#' "D")` for `cog_revenue()` -- see `.verb_spendrev()`). Passed through to
|
||||
#' `.attach_ig_counterparts()` to keep the intergovernmental-counterpart
|
||||
#' lookup scoped to the calling verb's own flow family.
|
||||
#' @param long_view Name of the verb's own long view (from
|
||||
#' `.select_long_view()`), passed through to `.suppressed_components()` to
|
||||
#' measure the second qualifying path (uscogdata#9).
|
||||
#' @return List of `list(recipe_id, label, available_years, hint,
|
||||
#' ig_recipe_id)`, possibly empty.
|
||||
#' ig_recipe_id, trigger, suppressed_amount, suppressed_years,
|
||||
#' suppressed_codes)`, possibly empty.
|
||||
#' @noRd
|
||||
.build_suggestions <- function(con, govid, years, category, result, basis,
|
||||
flow_prefixes) {
|
||||
flow_prefixes, long_view) {
|
||||
if (!identical(basis, "harmonized") || is.null(category)) return(list())
|
||||
|
||||
# Exclude any recipe that is ITSELF an intergovernmental (M/L) recipe --
|
||||
@@ -86,7 +98,14 @@
|
||||
unique(as.integer(result$year))
|
||||
}
|
||||
gap_years <- setdiff(as.integer(years), result_years)
|
||||
if (length(gap_years) == 0L) return(list())
|
||||
|
||||
# Path 2 (uscogdata#9): component dollars this government holds that the
|
||||
# verb's own view structurally excludes. Measured across ALL requested
|
||||
# years, not just gap years -- the whole point is that a year with rows can
|
||||
# still be missing dollars.
|
||||
supp <- .suppressed_components(con, candidates, govid, years, long_view)
|
||||
|
||||
if (length(gap_years) == 0L && nrow(supp) == 0L) return(list())
|
||||
|
||||
meta <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf(
|
||||
"SELECT recipe_id, any_value(label) AS label,
|
||||
@@ -97,35 +116,55 @@
|
||||
.sql_lit_chr(candidates)
|
||||
)))
|
||||
|
||||
# Which (recipe_id, year) pairs the recipe's own generic join actually
|
||||
# covers for this government, restricted to the gap years -- the same
|
||||
# join .run_recipe() uses (component year_min/year_max + gov_type_scope,
|
||||
# no is_aggregate filter), just checking existence instead of summing.
|
||||
covered <- DBI::dbGetQuery(con, sprintf(
|
||||
"SELECT DISTINCT r.recipe_id, l.year
|
||||
FROM long l
|
||||
JOIN harmonization_recipes r
|
||||
ON l.item_code = r.component_code
|
||||
AND l.year BETWEEN r.year_min AND r.year_max
|
||||
AND (r.gov_type_scope = 'all'
|
||||
OR (r.gov_type_scope = 'state' AND l.type = 0)
|
||||
OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
|
||||
WHERE r.recipe_id IN (%s)
|
||||
AND l.canonical_govid IN (%s)
|
||||
AND l.year IN (%s)",
|
||||
.sql_lit_chr(candidates), .sql_lit_chr(govid),
|
||||
paste(gap_years, collapse = ",")
|
||||
))
|
||||
# Path 1 (unchanged): (recipe, year) pairs the recipe's own generic join
|
||||
# covers for this government, restricted to the gap years.
|
||||
covered <- if (length(gap_years) == 0L) {
|
||||
data.frame(recipe_id = character(0), year = integer(0))
|
||||
} else {
|
||||
DBI::dbGetQuery(con, sprintf(
|
||||
"SELECT DISTINCT r.recipe_id, l.year
|
||||
FROM long l
|
||||
JOIN harmonization_recipes r
|
||||
ON l.item_code = r.component_code
|
||||
AND l.year BETWEEN r.year_min AND r.year_max
|
||||
AND (r.gov_type_scope = 'all'
|
||||
OR (r.gov_type_scope = 'state' AND l.type = 0)
|
||||
OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
|
||||
WHERE r.recipe_id IN (%s)
|
||||
AND l.canonical_govid IN (%s)
|
||||
AND l.year IN (%s)",
|
||||
.sql_lit_chr(candidates), .sql_lit_chr(govid),
|
||||
paste(gap_years, collapse = ",")
|
||||
))
|
||||
}
|
||||
|
||||
suggestions <- list()
|
||||
for (rid in candidates) {
|
||||
if (!rid %in% covered$recipe_id) next
|
||||
empty_hit <- rid %in% covered$recipe_id
|
||||
s_rows <- supp[supp$recipe_id == rid, , drop = FALSE]
|
||||
supp_hit <- nrow(s_rows) > 0L
|
||||
if (!empty_hit && !supp_hit) next
|
||||
m <- meta[meta$recipe_id == rid, ]
|
||||
suggestions[[length(suggestions) + 1L]] <- list(
|
||||
recipe_id = rid,
|
||||
label = m$label[[1]],
|
||||
available_years = c(as.integer(m$year_min), as.integer(m$year_max)),
|
||||
hint = sprintf("re-run with recipe = '%s'", rid)
|
||||
hint = sprintf("re-run with recipe = '%s'", rid),
|
||||
# An empty year is the stronger claim -- the category returned nothing
|
||||
# at all -- so it wins when both paths qualify. The suppressed_* fields
|
||||
# are still populated, so an empty_year fire also reports its dollars.
|
||||
trigger = if (empty_hit) "empty_year" else "suppressed_component",
|
||||
suppressed_amount = if (supp_hit) sum(s_rows$suppressed_amount) else 0,
|
||||
suppressed_years = if (supp_hit) {
|
||||
sort(unique(as.integer(s_rows$year)))
|
||||
} else {
|
||||
integer(0)
|
||||
},
|
||||
suppressed_codes = if (supp_hit) {
|
||||
sort(unique(unlist(strsplit(s_rows$suppressed_codes, ",", fixed = TRUE))))
|
||||
} else {
|
||||
character(0)
|
||||
}
|
||||
)
|
||||
}
|
||||
.attach_ig_counterparts(con, suggestions, flow_prefixes)
|
||||
|
||||
@@ -282,3 +282,99 @@ test_that(".suppressed_components rejects a long_view outside the allowlist", {
|
||||
years = 2011L, long_view = "long; DROP TABLE x"),
|
||||
class = "uscogdata_internal_error")
|
||||
})
|
||||
|
||||
test_that("uscogdata#9: Public Welfare signposts its suppressed E67/E68 dollars", {
|
||||
# The bug: E74/E79 return rows for FY2011, so there is no row-absence gap,
|
||||
# so nothing fired -- while E67 ($1,803,872,000) and E68 ($271,589,000) were
|
||||
# dropped for being aggregate-published. LA County reports $3,185,943,000
|
||||
# and omits $2,075,461,000, a 39% understatement, silently.
|
||||
skip_if_no_corpus()
|
||||
r <- suppressMessages(
|
||||
cog_spending("061037123085", years = 2011L, category = "Public Welfare"))
|
||||
sugg <- attr(r, "provenance")$suggestions
|
||||
|
||||
expect_length(sugg, 2L)
|
||||
ids <- vapply(sugg, function(s) s$recipe_id, character(1))
|
||||
expect_setequal(ids, c("welfare_cash_e67_wide", "welfare_cash_e68_wide"))
|
||||
|
||||
e67 <- sugg[[which(ids == "welfare_cash_e67_wide")]]
|
||||
expect_equal(e67$trigger, "suppressed_component")
|
||||
expect_equal(e67$suppressed_amount, 1803872000)
|
||||
expect_equal(e67$suppressed_years, 2011L)
|
||||
expect_equal(e67$suppressed_codes, "E67")
|
||||
expect_equal(e67$hint, "re-run with recipe = 'welfare_cash_e67_wide'")
|
||||
|
||||
e68 <- sugg[[which(ids == "welfare_cash_e68_wide")]]
|
||||
expect_equal(e68$trigger, "suppressed_component")
|
||||
expect_equal(e68$suppressed_amount, 271589000)
|
||||
expect_equal(e68$suppressed_codes, "E68")
|
||||
})
|
||||
|
||||
test_that("uscogdata#9: an empty_year fire keeps its trigger and gains the dollars", {
|
||||
# Corrections is the case that already worked: zero rows in FY2011, so the
|
||||
# row-absence path fires. It must keep firing, keep trigger = "empty_year",
|
||||
# keep its IG counterpart -- and now also report what was suppressed.
|
||||
skip_if_no_corpus()
|
||||
r <- suppressMessages(
|
||||
cog_spending("061037123085", years = 2011L, category = "Corrections"))
|
||||
sugg <- attr(r, "provenance")$suggestions
|
||||
|
||||
expect_length(sugg, 3L)
|
||||
ids <- vapply(sugg, function(s) s$recipe_id, character(1))
|
||||
expect_setequal(ids, c("corrections_combined", "corrections_capital_combined",
|
||||
"corrections_other_capital_combined"))
|
||||
expect_true(all(vapply(sugg, function(s) s$trigger, character(1)) == "empty_year"))
|
||||
|
||||
cc <- sugg[[which(ids == "corrections_combined")]]
|
||||
expect_equal(cc$suppressed_amount, 1371460000)
|
||||
expect_equal(cc$suppressed_codes, "E05")
|
||||
expect_equal(cc$ig_recipe_id, "corrections_ig_local_combined")
|
||||
})
|
||||
|
||||
test_that("uscogdata#9: the revenue verb inherits the same trigger", {
|
||||
# Alaska state FY2011 Miscellaneous Revenue reports $943,842,000 from
|
||||
# U11/U20/U30 while dropping $1,899,995,000 of aggregate-published `U4-`
|
||||
# rents and royalties -- the omission is LARGER than the reported figure.
|
||||
skip_if_no_corpus()
|
||||
r <- suppressMessages(
|
||||
cog_revenue("020000227749", years = 2011L,
|
||||
category = "Miscellaneous Revenue"))
|
||||
sugg <- attr(r, "provenance")$suggestions
|
||||
|
||||
expect_length(sugg, 1L)
|
||||
expect_equal(sugg[[1]]$recipe_id, "rents_royalties_u4_wide")
|
||||
expect_equal(sugg[[1]]$trigger, "suppressed_component")
|
||||
expect_equal(sugg[[1]]$suppressed_amount, 1899995000)
|
||||
expect_equal(sugg[[1]]$suppressed_codes, "U4-")
|
||||
# A revenue recipe must never be handed an M/L expenditure counterpart.
|
||||
expect_null(sugg[[1]]$ig_recipe_id)
|
||||
})
|
||||
|
||||
test_that("uscogdata#9: no partial-coverage fire in a modern year", {
|
||||
skip_if_no_corpus()
|
||||
r <- cog_spending("061037123085", years = 2019L, category = "Public Welfare")
|
||||
expect_length(attr(r, "provenance")$suggestions, 0L)
|
||||
})
|
||||
|
||||
test_that("uscogdata#9: leaf-and-classified wide-era families never fire", {
|
||||
# higher_ed_e18_wide and general_gov_e89_wide are the control group: their
|
||||
# components (E16/E18, E85/E89) are ordinary classified leaves even in the
|
||||
# wide era, so widening the trigger must leave them silent. This is the
|
||||
# measurement that refutes "it would fire on every category in every legacy
|
||||
# year" -- corpus-wide on the fixture, these two produce zero suppressed rows.
|
||||
skip_if_no_corpus()
|
||||
con <- uscogdata:::.ensure_session()
|
||||
n <- DBI::dbGetQuery(con,
|
||||
"SELECT COUNT(*) AS n
|
||||
FROM long l
|
||||
JOIN harmonization_recipes r
|
||||
ON l.item_code = r.component_code
|
||||
AND l.year BETWEEN r.year_min AND r.year_max
|
||||
WHERE r.recipe_id IN ('higher_ed_e18_wide', 'general_gov_e89_wide')
|
||||
AND l.amt <> 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM spending_long_harmonized v
|
||||
WHERE v.canonical_govid = l.canonical_govid
|
||||
AND v.year = l.year AND v.item_code = l.item_code)")$n
|
||||
expect_equal(as.integer(n), 0L)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user