jared
4de915b557
Adds cog_recipes() to list the curated harmonization_recipes catalog (24 recipes / schema_version >= 5), and a recipe= argument on cog_spending()/ cog_revenue() that runs a recipe's generic multi-code join instead of the category view: SUM(amt * weight) across whichever component codes are present for a (year, canonical_govid), scoped by gov_type_scope. The join deliberately does not filter is_aggregate -- the wide era (<= 2011) exposes these split families (corrections 04+05, IG *89/*47, U4- rents, etc.) ONLY as aggregate rows, with leaf codes first appearing in 2012, so excluding aggregates would zero out the wide-era half of every recipe. This is safe by corpus construction: wide-era rows are aggregate-only, modern rows are leaf-only, and every component is year-scoped, so there is no double-counting. recipe= is mutually exclusive with category=; the result's subtype column reads "recipe" and category reads the recipe's label. Adds recipe-component-driven signposting: when a basis="harmonized" + category query comes back with zero rows in a requested year, and a harmonization recipe covering that category would actually produce rows for this government in that year (via the same join .run_recipe() uses), the recipe is surfaced in provenance$suggestions plus one cli::cli_inform() message. This is deliberately keyed off recipe components rather than harmonization_map's suggested_recipe_id column (which is empty on every live row -- the wide era's split families are NA-by-construction via aggregate exclusion, not an NA ruling to hang a suggestion off of). Also populates the previously-always-empty provenance$series_break_refs (schema v5 only: series_breaks_pq rows whose fin_code is among the observed codes and whose break_year falls in the requested span), and extends cog_explain() with Basis/Harmonization/Recipe/Suggestions/Series breaks sections.
120 lines
5.1 KiB
R
120 lines
5.1 KiB
R
# R/suggestions.R
|
|
# Recipe-component-driven signposting: when a basis = "harmonized" query for
|
|
# a category comes back with a coverage gap in some requested years (the
|
|
# result has no rows at all in that year) that a harmonization recipe would
|
|
# actually fill for this government, surface that recipe as a suggestion.
|
|
#
|
|
# This is deliberately keyed off the recipe catalog's component codes, not
|
|
# off harmonization_map rows: no live map row carries a non-blank
|
|
# suggested_recipe_id (the corpus's wide era exposes split families like
|
|
# corrections functions 04+05 ONLY as aggregate rows, which basis =
|
|
# "harmonized" excludes by construction -- there's no NA ruling to hang a
|
|
# suggestion off of, just a leaf-code absence a recipe happens to fill).
|
|
# See docs/phase_r_harmonization_review.md § 0.3.
|
|
#
|
|
# Scope is deliberately narrow: signposting only runs when the caller
|
|
# supplied a `category` (an un-scoped, all-categories query has no single
|
|
# coverage question to answer) and only flags a recipe when the ACTUAL
|
|
# result has zero rows in a requested year AND the candidate recipe's own
|
|
# generic join (same join .run_recipe() uses, including its wide-era
|
|
# aggregate rows) produces at least one row for this government in that
|
|
# year. Checking presence per-government (not corpus-wide) avoids false
|
|
# positives from ordinary reporting variance -- most governments don't use
|
|
# every sibling code in a multi-code category every year, and that is not
|
|
# a format-boundary gap worth signposting.
|
|
|
|
#' Build the `prov$suggestions` list for a (non-recipe) basis = "harmonized"
|
|
#' verb call: recipes whose generic join would fill a real gap in `result`.
|
|
#'
|
|
#' @param con Active DuckDB connection.
|
|
#' @param govid Character vector of canonical_govid values (the verb's raw
|
|
#' `govid`).
|
|
#' @param years Integer vector of requested years.
|
|
#' @param category `category` argument as passed to the verb (character
|
|
#' vector or `NULL`; suggestions are only computed when non-NULL).
|
|
#' @param result The verb's already-computed result tibble (post basis
|
|
#' query, pre per_capita/adjust_to_year).
|
|
#' @param basis The *resolved* basis (`"harmonized"` or `"raw"`).
|
|
#' @return List of `list(recipe_id, label, available_years, hint)`, possibly
|
|
#' empty.
|
|
#' @noRd
|
|
.build_suggestions <- function(con, govid, years, category, result, basis) {
|
|
if (!identical(basis, "harmonized") || is.null(category)) return(list())
|
|
|
|
candidates <- DBI::dbGetQuery(con, sprintf(
|
|
"SELECT DISTINCT recipe_id FROM harmonization_recipes
|
|
WHERE component_code IN (
|
|
SELECT DISTINCT item_code FROM summary_categories WHERE category IN (%s)
|
|
)",
|
|
.sql_lit_chr(category)
|
|
))$recipe_id
|
|
if (length(candidates) == 0L) return(list())
|
|
|
|
result_years <- if (is.null(result) || nrow(result) == 0L) {
|
|
integer(0)
|
|
} else {
|
|
unique(as.integer(result$year))
|
|
}
|
|
gap_years <- setdiff(as.integer(years), result_years)
|
|
if (length(gap_years) == 0L) return(list())
|
|
|
|
meta <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf(
|
|
"SELECT recipe_id, any_value(label) AS label,
|
|
MIN(year_min) AS year_min, MAX(year_max) AS year_max
|
|
FROM harmonization_recipes
|
|
WHERE recipe_id IN (%s)
|
|
GROUP BY recipe_id",
|
|
.sql_lit_chr(candidates)
|
|
)))
|
|
|
|
# Which (recipe_id, year) pairs the recipe's own generic join actually
|
|
# covers for this government, restricted to the gap years -- the same
|
|
# join .run_recipe() uses (component year_min/year_max + gov_type_scope,
|
|
# no is_aggregate filter), just checking existence instead of summing.
|
|
covered <- DBI::dbGetQuery(con, sprintf(
|
|
"SELECT DISTINCT r.recipe_id, l.year
|
|
FROM long l
|
|
JOIN harmonization_recipes r
|
|
ON l.item_code = r.component_code
|
|
AND l.year BETWEEN r.year_min AND r.year_max
|
|
AND (r.gov_type_scope = 'all'
|
|
OR (r.gov_type_scope = 'state' AND l.type = 0)
|
|
OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
|
|
WHERE r.recipe_id IN (%s)
|
|
AND l.canonical_govid IN (%s)
|
|
AND l.year IN (%s)",
|
|
.sql_lit_chr(candidates), .sql_lit_chr(govid),
|
|
paste(gap_years, collapse = ",")
|
|
))
|
|
|
|
suggestions <- list()
|
|
for (rid in candidates) {
|
|
if (!rid %in% covered$recipe_id) next
|
|
m <- meta[meta$recipe_id == rid, ]
|
|
suggestions[[length(suggestions) + 1L]] <- list(
|
|
recipe_id = rid,
|
|
label = m$label[[1]],
|
|
available_years = c(as.integer(m$year_min), as.integer(m$year_max)),
|
|
hint = sprintf("re-run with recipe = '%s'", rid)
|
|
)
|
|
}
|
|
suggestions
|
|
}
|
|
|
|
#' Emit the single cli::cli_inform() message summarizing all suggestions
|
|
#' for a verb call (the brief's "one message", not one per suggestion).
|
|
#' Bullet text is pre-formatted plain text (no cli/glue `{}` markup) since
|
|
#' recipe ids/labels are untrusted-ish data values, not literal call-site
|
|
#' expressions.
|
|
#' @noRd
|
|
.inform_suggestions <- function(suggestions) {
|
|
bullets <- vapply(suggestions, function(s) {
|
|
sprintf("%s (%d-%d): %s", s$recipe_id,
|
|
s$available_years[1], s$available_years[2], s$hint)
|
|
}, character(1))
|
|
cli::cli_inform(c(
|
|
i = "Coverage gap detected for the requested years; a harmonization recipe may fill it:",
|
|
stats::setNames(bullets, rep("*", length(bullets)))
|
|
))
|
|
}
|