feat: basis= harmonized/raw with v4/v5 dual-accept
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.
Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.
Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
@@ -26,3 +26,34 @@ with_fixture_corpus <- function(code) {
|
||||
}, add = TRUE)
|
||||
force(code)
|
||||
}
|
||||
|
||||
# Copy the bundled fixture to a temp dir with manifest.json's schema_version
|
||||
# patched to `version`, then run `code` against it with a clean session
|
||||
# (mirrors with_fixture_corpus()). Used to exercise the v4/v5 dual-accept
|
||||
# path without a second physical fixture tree: a real v4 corpus has no
|
||||
# harmonization_map/harmonization_recipes/series_breaks parquet files, but
|
||||
# .register_views() only *reads* those when schema_version >= 5 (see
|
||||
# R/views.R), so a doctored copy of the (v5) bundled fixture with the
|
||||
# manifest's schema_version knocked down to 4 is a faithful stand-in.
|
||||
with_doctored_schema_version <- function(version, code) {
|
||||
src <- fixture_corpus_path()
|
||||
tmp <- withr::local_tempdir(.local_envir = parent.frame())
|
||||
file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
|
||||
|
||||
manifest_path <- file.path(tmp, "manifest.json")
|
||||
m <- jsonlite::fromJSON(manifest_path, simplifyVector = FALSE)
|
||||
m$schema_version <- as.integer(version)
|
||||
writeLines(
|
||||
jsonlite::toJSON(m, auto_unbox = TRUE, pretty = TRUE, null = "null"),
|
||||
manifest_path
|
||||
)
|
||||
|
||||
old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
|
||||
uscogdata:::cog_close()
|
||||
Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
|
||||
on.exit({
|
||||
uscogdata:::cog_close()
|
||||
if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
|
||||
}, add = TRUE)
|
||||
force(code)
|
||||
}
|
||||
|
||||
@@ -107,6 +107,44 @@ test_that("cog_manifest returns the active session's parsed manifest", {
|
||||
expect_true(m$schema_version >= 4L)
|
||||
yrs <- vapply(m$files$long_partitions, function(p) as.integer(p$year),
|
||||
integer(1))
|
||||
expect_setequal(yrs, c(2019L, 2020L))
|
||||
expect_setequal(yrs, c(2011L, 2012L, 2019L, 2020L))
|
||||
})
|
||||
})
|
||||
|
||||
test_that(".validate_schema accepts schema_version 4 and 5, rejects others", {
|
||||
expect_silent(uscogdata:::.validate_schema(list(schema_version = 4L)))
|
||||
expect_silent(uscogdata:::.validate_schema(list(schema_version = 5L)))
|
||||
expect_error(
|
||||
uscogdata:::.validate_schema(list(schema_version = 3L)),
|
||||
"schema_version"
|
||||
)
|
||||
expect_error(
|
||||
uscogdata:::.validate_schema(list(schema_version = 6L)),
|
||||
"schema_version"
|
||||
)
|
||||
})
|
||||
|
||||
test_that("cog_open succeeds against a doctored schema_version 4 corpus (dual-accept)", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
con <- cog_open()
|
||||
expect_true(DBI::dbIsValid(con))
|
||||
expect_equal(as.integer(cog_manifest()$schema_version), 4L)
|
||||
|
||||
# Core (pre-Phase-R2) views must still register on a v4 corpus.
|
||||
views <- DBI::dbGetQuery(con,
|
||||
"SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'main' AND table_type = 'VIEW'"
|
||||
)$table_name
|
||||
expect_true(all(c("spending_annotated", "revenue_annotated") %in% views))
|
||||
|
||||
# Schema-v5-only harmonization views must NOT register on a v4 corpus:
|
||||
# their parquet sources don't exist there and DuckDB's read_parquet()
|
||||
# errors eagerly at CREATE VIEW time for a missing file/glob, so
|
||||
# .register_views() gates these on manifest$schema_version >= 5.
|
||||
expect_false(any(c(
|
||||
"spending_long_harmonized", "spending_annotated_harmonized",
|
||||
"harmonization_recipes", "harmonization_map", "series_breaks_pq"
|
||||
) %in% views))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -36,3 +36,20 @@ test_that("cog_revenue result has provenance attribute", {
|
||||
test_that("cog_revenue rejects invalid inputs", {
|
||||
expect_error(cog_revenue(list(), 2020L), "character|data frame")
|
||||
})
|
||||
|
||||
test_that("cog_revenue basis = 'harmonized' (default) matches 'raw' in this fixture window", {
|
||||
skip_if_no_corpus()
|
||||
r_raw <- cog_revenue("121011212191", 2019:2020, basis = "raw")
|
||||
r_harm <- cog_revenue("121011212191", 2019:2020, basis = "harmonized")
|
||||
expect_equal(attr(r_raw, "provenance")$basis, "raw")
|
||||
expect_equal(attr(r_harm, "provenance")$basis, "harmonized")
|
||||
expect_equal(sum(r_raw$amt_nominal), sum(r_harm$amt_nominal))
|
||||
})
|
||||
|
||||
test_that("cog_revenue provenance carries the harmonization block", {
|
||||
skip_if_no_corpus()
|
||||
r <- cog_revenue("121011212191", 2020L)
|
||||
h <- attr(r, "provenance")$harmonization
|
||||
expect_true(h$applied)
|
||||
expect_true(h$na_rows_excluded >= 0L)
|
||||
})
|
||||
|
||||
@@ -189,3 +189,114 @@ test_that("provenance records per-year denominator metadata", {
|
||||
expect_equal(length(pc$popyear_range), 2L)
|
||||
})
|
||||
})
|
||||
|
||||
# --- basis = "harmonized" / "raw" (Phase R2, schema v5) --------------------
|
||||
|
||||
test_that("basis = 'raw' reproduces the pre-harmonization Broward Police totals", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", years = 2019:2020, category = "Police",
|
||||
basis = "raw")
|
||||
# Regression pin captured against the schema v5 fixture (2026-07-18,
|
||||
# pipeline_commit ece9b32) before basis = "harmonized" existed as a
|
||||
# concept; these are the same totals the pre-Phase-R2 default query
|
||||
# returned (spending_annotated is untouched by the harmonized views).
|
||||
ops <- r$amt_nominal[r$year == 2019L & r$spend_subtype == "operations"]
|
||||
cap <- r$amt_nominal[r$year == 2020L & r$spend_subtype == "capital"]
|
||||
expect_equal(ops, 483560000)
|
||||
expect_equal(cap, 26693000)
|
||||
expect_equal(attr(r, "provenance")$basis, "raw")
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis = 'harmonized' (default) matches 'raw' when no harmonization rule applies", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
# Every `method = "collapse"` mapping in the curated harmonization_map
|
||||
# ends by FY2004 for codes inside the spending/revenue flow-type
|
||||
# prefixes (E/F/G/K, T/A/U/B/C/D); the one collapse extending to FY2011
|
||||
# (L38/M38 -> L36/M36) is intergovernmental-transfer (L/M prefix) codes
|
||||
# that were never part of spending_long/revenue_long to begin with. So
|
||||
# for the fixture's 2011-2020 window, basis = "harmonized" is a
|
||||
# data-verified no-op vs "raw" for in-scope codes -- this is the
|
||||
# positive-control counterpart to the synthetic REPLACE-mechanism test
|
||||
# in test-views.R, which proves the fold itself works when data exists.
|
||||
r_raw <- cog_spending("121011212191", c(2011L, 2012L, 2019L, 2020L),
|
||||
"Police", basis = "raw")
|
||||
r_harm <- cog_spending("121011212191", c(2011L, 2012L, 2019L, 2020L),
|
||||
"Police", basis = "harmonized")
|
||||
expect_equal(attr(r_harm, "provenance")$basis, "harmonized")
|
||||
expect_equal(
|
||||
r_harm$amt_nominal[order(r_harm$year, r_harm$spend_subtype)],
|
||||
r_raw$amt_nominal[order(r_raw$year, r_raw$spend_subtype)]
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis defaults to 'harmonized' when not passed", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2020L, "Police")
|
||||
expect_equal(attr(r, "provenance")$basis, "harmonized")
|
||||
})
|
||||
})
|
||||
|
||||
test_that("provenance carries basis + harmonization block with na_rows_excluded", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2011:2012, "Corrections")
|
||||
prov <- attr(r, "provenance")
|
||||
expect_equal(prov$basis, "harmonized")
|
||||
expect_true(prov$harmonization$applied)
|
||||
expect_true(prov$harmonization$na_rows_excluded >= 0L)
|
||||
expect_true(prov$harmonization$na_amount_excluded >= 0)
|
||||
# Data-verified for this fixture: none of the discontinued_na rulings
|
||||
# (S74, Z61, X04, X06, the debt-detail family, L24) fall inside the
|
||||
# E/F/G/K spending prefixes, so the exclusion count is exactly zero for
|
||||
# every year in the bundled window -- see
|
||||
# docs/phase_r_harmonization_review.md § 1.3/1.4.
|
||||
expect_equal(prov$harmonization$na_rows_excluded, 0L)
|
||||
expect_equal(prov$harmonization$na_amount_excluded, 0)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("basis = 'raw' never populates the harmonization exclusion block", {
|
||||
skip_if_no_corpus()
|
||||
with_fixture_corpus({
|
||||
r <- cog_spending("121011212191", 2020L, "Corrections", basis = "raw")
|
||||
h <- attr(r, "provenance")$harmonization
|
||||
expect_false(h$applied)
|
||||
expect_equal(h$na_rows_excluded, 0L)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: basis silently resolves to raw (default) with a provenance note", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
r <- cog_spending("121011212191", 2019L, "Police")
|
||||
prov <- attr(r, "provenance")
|
||||
expect_equal(prov$basis, "raw")
|
||||
expect_match(prov$basis_note, "raw", fixed = TRUE)
|
||||
expect_match(prov$basis_note, "schema_version", fixed = TRUE)
|
||||
expect_false(prov$harmonization$applied)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: explicit basis = 'harmonized' aborts", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
expect_error(
|
||||
cog_spending("121011212191", 2019L, "Police", basis = "harmonized"),
|
||||
class = "uscogdata_basis_unsupported"
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
test_that("v4 corpus: explicit basis = 'raw' still works", {
|
||||
skip_if_no_corpus()
|
||||
with_doctored_schema_version(4L, {
|
||||
r <- cog_spending("121011212191", 2019L, "Police", basis = "raw")
|
||||
expect_equal(attr(r, "provenance")$basis, "raw")
|
||||
expect_gt(nrow(r), 0L)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -14,6 +14,62 @@ test_that("all expected views register on session open", {
|
||||
expect_true(all(expected %in% views$table_name))
|
||||
})
|
||||
|
||||
test_that("the REPLACE(harmonized_code AS item_code) pattern folds a collapsed code", {
|
||||
# spending_long_harmonized / revenue_long_harmonized (inst/sql/22-, 23-)
|
||||
# are defined as:
|
||||
# SELECT * REPLACE (harmonized_code AS item_code) FROM long WHERE ...
|
||||
# None of the curated harmonization_map's `collapse` rulings land inside
|
||||
# the bundled fixture's 2011-2020 window for spending/revenue-prefixed
|
||||
# codes (see the "basis = 'harmonized' (default) matches 'raw'" test in
|
||||
# test-spending.R and docs/phase_r_harmonization_review.md § 0.2/§ 2), so
|
||||
# there is no real fixture row that exercises a nonzero fold. This test
|
||||
# proves the REPLACE mechanism itself is correct against a synthetic
|
||||
# long-shaped table with a deliberate E38 -> E36 collapse, independent of
|
||||
# whether the bundled data happens to contain one right now.
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
on.exit(cog_close())
|
||||
|
||||
DBI::dbExecute(con, "
|
||||
CREATE OR REPLACE TEMP TABLE synthetic_long AS
|
||||
SELECT * FROM (VALUES
|
||||
('121011212191', 2004, 'E36', 70942668, false, 'E36'),
|
||||
('121011212191', 2004, 'E38', 837485, false, 'E36'),
|
||||
('121011212191', 2004, 'E62', 100000, false, 'E62')
|
||||
) AS t(canonical_govid, year, item_code, amt, is_aggregate, harmonized_code)
|
||||
")
|
||||
|
||||
folded <- DBI::dbGetQuery(con, "
|
||||
SELECT item_code, SUM(amt) AS amt
|
||||
FROM (SELECT * REPLACE (harmonized_code AS item_code) FROM synthetic_long)
|
||||
GROUP BY item_code ORDER BY item_code
|
||||
")
|
||||
expect_setequal(folded$item_code, c("E36", "E62"))
|
||||
expect_equal(folded$amt[folded$item_code == "E36"], 70942668 + 837485)
|
||||
expect_equal(folded$amt[folded$item_code == "E62"], 100000)
|
||||
|
||||
DBI::dbExecute(con, "DROP TABLE synthetic_long")
|
||||
})
|
||||
|
||||
test_that("schema v5 harmonization views register when the corpus supports them", {
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
on.exit(cog_close())
|
||||
manifest <- uscogdata:::.uscogdata_env$manifest
|
||||
skip_if(as.integer(manifest$schema_version) < 5L, "fixture is schema_version < 5")
|
||||
|
||||
views <- DBI::dbGetQuery(con,
|
||||
"SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'main' AND table_type = 'VIEW'"
|
||||
)
|
||||
expected_v5 <- c(
|
||||
"spending_long_harmonized", "revenue_long_harmonized",
|
||||
"spending_annotated_harmonized", "revenue_annotated_harmonized",
|
||||
"harmonization_map", "harmonization_recipes", "series_breaks_pq"
|
||||
)
|
||||
expect_true(all(expected_v5 %in% views$table_name))
|
||||
})
|
||||
|
||||
test_that("spending_long filters to E/F/G/K prefixes and excludes aggregates", {
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
@@ -69,13 +125,15 @@ test_that("gov_population_yearly exposes one row per (year, canonical_govid)", {
|
||||
WHERE canonical_govid = '121011212191'
|
||||
ORDER BY year"
|
||||
)
|
||||
expect_setequal(df$year, c(2019L, 2020L))
|
||||
expect_equal(nrow(df), 2L)
|
||||
expect_setequal(df$year, c(2011L, 2012L, 2019L, 2020L))
|
||||
expect_equal(nrow(df), 4L)
|
||||
expect_true(all(!is.na(df$population)))
|
||||
# Hardcoded values are from the bundled fixture (regenerated 2026-07-11
|
||||
# against cog_pipeline publish tree, pipeline_commit 1a00925, Phase P
|
||||
# schema_version 4). Update if the fixture is rebuilt against a
|
||||
# different source vintage.
|
||||
# Hardcoded values are from the bundled fixture (regenerated 2026-07-18
|
||||
# against cog_pipeline publish tree, pipeline_commit ece9b32, Phase R2
|
||||
# schema_version 5, years 2011/2012/2019/2020). Update if the fixture is
|
||||
# rebuilt against a different source vintage.
|
||||
expect_equal(df$population[df$year == 2011L], 1759591L)
|
||||
expect_equal(df$population[df$year == 2012L], 1819773L)
|
||||
expect_equal(df$population[df$year == 2019L], 1935878L)
|
||||
expect_equal(df$population[df$year == 2020L], 1952778L)
|
||||
# Uniqueness on (year, canonical_govid) across the whole view.
|
||||
|
||||
Reference in New Issue
Block a user