diff --git a/DESCRIPTION b/DESCRIPTION index 7e248c8..58214e1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -30,6 +30,6 @@ Suggests: ggplot2 Config/testthat/edition: 3 VignetteBuilder: knitr -RoxygenNote: 7.3.3 MinCorpusSchema: 4 MaxCorpusSchema: 5 +Config/roxygen2/version: 8.0.0 diff --git a/NEWS.md b/NEWS.md index 88e8de2..ca4aaf8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,27 @@ # uscogdata 0.1.0 (development) +## Signposting now catches partially-suppressed categories + +* A coverage suggestion used to fire only when a category returned **no rows + at all** in a requested year. That missed the more dangerous case: a + category that still returns rows while silently dropping component codes + the wide era publishes only as aggregates (#9). `cog_spending(category = + "Public Welfare")` for FY2011 returned a plausible figure that omitted + `E67`/`E68` entirely -- for Los Angeles County, $2,075,461,000 of a true + $5,261,404,000, a 39% understatement, with `provenance$suggestions` empty. +* Suggestions now also fire on **partial** coverage, and every suggestion + carries `trigger` (`"empty_year"` or `"suppressed_component"`), + `suppressed_amount`, `suppressed_years` and `suppressed_codes`, so a caller + can see how much is missing and decide whether to re-run with the recipe. +* `cog_revenue()` gets the same fix through the shared verb path. Alaska's + FY2011 `Miscellaneous Revenue` reported $943,842,000 while dropping + $1,899,995,000 of aggregate-published `U4-` rents and royalties. +* The trigger stays recipe-driven, so it only fires where a harmonization + recipe actually exists to name the fix. Measured on the bundled fixture, + every fire lands in the wide era; `higher_ed_e18_wide` and + `general_gov_e89_wide` stay silent, because their components are ordinary + classified leaves even pre-2012. + ## New: `cog_balances()` for cash-and-security holdings * New `cog_balances()` exposes the 14 cash-and-security holding codes diff --git a/inst/schemas/provenance-v1.json b/inst/schemas/provenance-v1.json index b7e942b..6bf15ec 100644 --- a/inst/schemas/provenance-v1.json +++ b/inst/schemas/provenance-v1.json @@ -33,7 +33,48 @@ }, "harmonization": { "type": "object" }, "recipe": { "type": ["object", "null"] }, - "suggestions": { "type": "array" }, + "suggestions": { + "type": "array", + "description": "Harmonization recipes that would fill incomplete coverage in the requested years for this government. Empty on a healthy query, on an un-scoped (category = NULL) query, on basis = 'raw', and on a recipe = query (which resolves its own coverage).", + "items": { + "type": "object", + "required": ["recipe_id", "label", "available_years", "hint", "trigger", + "suppressed_amount", "suppressed_years", "suppressed_codes"], + "properties": { + "recipe_id": { "type": "string" }, + "label": { "type": "string" }, + "available_years": { + "type": "array", + "items": { "type": "integer" }, + "description": "[year_min, year_max] of the recipe's component coverage." + }, + "hint": { "type": "string" }, + "ig_recipe_id": { + "type": ["string", "null"], + "description": "The intergovernmental (M/L) counterpart recipe covering the same function suffixes, or null. Never set for revenue recipes." + }, + "trigger": { + "type": "string", + "enum": ["empty_year", "suppressed_component"], + "description": "Why this fired. 'empty_year': the result has no rows at all in a requested year. 'suppressed_component': the result HAS rows, but a component code carries dollars the verb's long view structurally excludes -- aggregate-published, or absent from summary_categories. 'empty_year' wins when both apply, being the stronger claim; the suppressed_* fields are populated either way." + }, + "suppressed_amount": { + "type": "number", + "description": "Full US dollars this government holds in the recipe's component codes that the result excludes, summed across the requested years. 0 when nothing is suppressed." + }, + "suppressed_years": { + "type": "array", + "items": { "type": "integer" }, + "description": "The requested years contributing to suppressed_amount." + }, + "suppressed_codes": { + "type": "array", + "items": { "type": "string" }, + "description": "The excluded component item codes, sorted." + } + } + } + }, "scope": { "type": "object" }, "codes_summed": { "type": "object" }, "aggregate_fallback": { "type": ["object", "null"] }, diff --git a/tests/testthat/test-recipes.R b/tests/testthat/test-recipes.R index 9f6ae57..9d4ec1f 100644 --- a/tests/testthat/test-recipes.R +++ b/tests/testthat/test-recipes.R @@ -404,3 +404,14 @@ test_that("uscogdata#9: cog_explain() reports the suppressed dollars", { out <- paste(testthat::capture_messages(cog_explain(r)), collapse = "") expect_match(out, "271,589,000", fixed = TRUE) }) + +test_that("the provenance schema documents the suggestion trigger fields", { + sch <- jsonlite::fromJSON( + system.file("schemas", "provenance-v1.json", package = "uscogdata"), + simplifyVector = FALSE) + props <- sch$properties$suggestions$items$properties + expect_true(all(c("trigger", "suppressed_amount", "suppressed_years", + "suppressed_codes") %in% names(props))) + expect_setequal(unlist(props$trigger$enum), + c("empty_year", "suppressed_component")) +})