feat: basis= harmonized/raw with v4/v5 dual-accept
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.
Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.
Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
@@ -189,3 +189,114 @@ test_that("provenance records per-year denominator metadata", {
|
||||
expect_equal(length(pc$popyear_range), 2L)
|
||||
})
|
||||
})
|
||||
|
||||
# --- basis = "harmonized" / "raw" (Phase R2, schema v5) --------------------
|
||||
|
||||
test_that("basis = 'raw' reproduces the pre-harmonization Broward Police totals", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", years = 2019:2020, category = "Police",
|
||||
basis = "raw")
|
||||
# Regression pin captured against the schema v5 fixture (2026-07-18,
|
||||
# pipeline_commit ece9b32) before basis = "harmonized" existed as a
|
||||
# concept; these are the same totals the pre-Phase-R2 default query
|
||||
# returned (spending_annotated is untouched by the harmonized views).
|
||||
ops <- r$amt_nominal[r$year == 2019L & r$spend_subtype == "operations"]
|
||||
cap <- r$amt_nominal[r$year == 2020L & r$spend_subtype == "capital"]
|
||||
expect_equal(ops, 483560000)
|
||||
expect_equal(cap, 26693000)
|
||||
expect_equal(attr(r, "provenance")$basis, "raw")
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis = 'harmonized' (default) matches 'raw' when no harmonization rule applies", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
# Every `method = "collapse"` mapping in the curated harmonization_map
|
||||
# ends by FY2004 for codes inside the spending/revenue flow-type
|
||||
# prefixes (E/F/G/K, T/A/U/B/C/D); the one collapse extending to FY2011
|
||||
# (L38/M38 -> L36/M36) is intergovernmental-transfer (L/M prefix) codes
|
||||
# that were never part of spending_long/revenue_long to begin with. So
|
||||
# for the fixture's 2011-2020 window, basis = "harmonized" is a
|
||||
# data-verified no-op vs "raw" for in-scope codes -- this is the
|
||||
# positive-control counterpart to the synthetic REPLACE-mechanism test
|
||||
# in test-views.R, which proves the fold itself works when data exists.
|
||||
r_raw <- cog_spending("121011212191", c(2011L, 2012L, 2019L, 2020L),
|
||||
"Police", basis = "raw")
|
||||
r_harm <- cog_spending("121011212191", c(2011L, 2012L, 2019L, 2020L),
|
||||
"Police", basis = "harmonized")
|
||||
expect_equal(attr(r_harm, "provenance")$basis, "harmonized")
|
||||
expect_equal(
|
||||
r_harm$amt_nominal[order(r_harm$year, r_harm$spend_subtype)],
|
||||
r_raw$amt_nominal[order(r_raw$year, r_raw$spend_subtype)]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis defaults to 'harmonized' when not passed", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2020L, "Police")
|
||||
expect_equal(attr(r, "provenance")$basis, "harmonized")
|
||||
})
|
||||
})
|
||||
|
||||
test_that("provenance carries basis + harmonization block with na_rows_excluded", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2011:2012, "Corrections")
|
||||
prov <- attr(r, "provenance")
|
||||
expect_equal(prov$basis, "harmonized")
|
||||
expect_true(prov$harmonization$applied)
|
||||
expect_true(prov$harmonization$na_rows_excluded >= 0L)
|
||||
expect_true(prov$harmonization$na_amount_excluded >= 0)
|
||||
# Data-verified for this fixture: none of the discontinued_na rulings
|
||||
# (S74, Z61, X04, X06, the debt-detail family, L24) fall inside the
|
||||
# E/F/G/K spending prefixes, so the exclusion count is exactly zero for
|
||||
# every year in the bundled window -- see
|
||||
# docs/phase_r_harmonization_review.md § 1.3/1.4.
|
||||
expect_equal(prov$harmonization$na_rows_excluded, 0L)
|
||||
expect_equal(prov$harmonization$na_amount_excluded, 0)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis = 'raw' never populates the harmonization exclusion block", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2020L, "Corrections", basis = "raw")
|
||||
h <- attr(r, "provenance")$harmonization
|
||||
expect_false(h$applied)
|
||||
expect_equal(h$na_rows_excluded, 0L)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: basis silently resolves to raw (default) with a provenance note", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
r <- cog_spending("121011212191", 2019L, "Police")
|
||||
prov <- attr(r, "provenance")
|
||||
expect_equal(prov$basis, "raw")
|
||||
expect_match(prov$basis_note, "raw", fixed = TRUE)
|
||||
expect_match(prov$basis_note, "schema_version", fixed = TRUE)
|
||||
expect_false(prov$harmonization$applied)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: explicit basis = 'harmonized' aborts", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
expect_error(
|
||||
cog_spending("121011212191", 2019L, "Police", basis = "harmonized"),
|
||||
class = "uscogdata_basis_unsupported"
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: explicit basis = 'raw' still works", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
r <- cog_spending("121011212191", 2019L, "Police", basis = "raw")
|
||||
expect_equal(attr(r, "provenance")$basis, "raw")
|
||||
expect_gt(nrow(r), 0L)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user