feat: basis= harmonized/raw with v4/v5 dual-accept
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.
Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.
Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
@@ -14,6 +14,62 @@ test_that("all expected views register on session open", {
|
||||
expect_true(all(expected %in% views$table_name))
|
||||
})
|
||||
|
||||
test_that("the REPLACE(harmonized_code AS item_code) pattern folds a collapsed code", {
|
||||
# spending_long_harmonized / revenue_long_harmonized (inst/sql/22-, 23-)
|
||||
# are defined as:
|
||||
# SELECT * REPLACE (harmonized_code AS item_code) FROM long WHERE ...
|
||||
# None of the curated harmonization_map's `collapse` rulings land inside
|
||||
# the bundled fixture's 2011-2020 window for spending/revenue-prefixed
|
||||
# codes (see the "basis = 'harmonized' (default) matches 'raw'" test in
|
||||
# test-spending.R and docs/phase_r_harmonization_review.md § 0.2/§ 2), so
|
||||
# there is no real fixture row that exercises a nonzero fold. This test
|
||||
# proves the REPLACE mechanism itself is correct against a synthetic
|
||||
# long-shaped table with a deliberate E38 -> E36 collapse, independent of
|
||||
# whether the bundled data happens to contain one right now.
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
on.exit(cog_close())
|
||||
|
||||
DBI::dbExecute(con, "
|
||||
CREATE OR REPLACE TEMP TABLE synthetic_long AS
|
||||
SELECT * FROM (VALUES
|
||||
('121011212191', 2004, 'E36', 70942668, false, 'E36'),
|
||||
('121011212191', 2004, 'E38', 837485, false, 'E36'),
|
||||
('121011212191', 2004, 'E62', 100000, false, 'E62')
|
||||
) AS t(canonical_govid, year, item_code, amt, is_aggregate, harmonized_code)
|
||||
")
|
||||
|
||||
folded <- DBI::dbGetQuery(con, "
|
||||
SELECT item_code, SUM(amt) AS amt
|
||||
FROM (SELECT * REPLACE (harmonized_code AS item_code) FROM synthetic_long)
|
||||
GROUP BY item_code ORDER BY item_code
|
||||
")
|
||||
expect_setequal(folded$item_code, c("E36", "E62"))
|
||||
expect_equal(folded$amt[folded$item_code == "E36"], 70942668 + 837485)
|
||||
expect_equal(folded$amt[folded$item_code == "E62"], 100000)
|
||||
|
||||
DBI::dbExecute(con, "DROP TABLE synthetic_long")
|
||||
})
|
||||
|
||||
test_that("schema v5 harmonization views register when the corpus supports them", {
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
on.exit(cog_close())
|
||||
manifest <- uscogdata:::.uscogdata_env$manifest
|
||||
skip_if(as.integer(manifest$schema_version) < 5L, "fixture is schema_version < 5")
|
||||
|
||||
views <- DBI::dbGetQuery(con,
|
||||
"SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'main' AND table_type = 'VIEW'"
|
||||
)
|
||||
expected_v5 <- c(
|
||||
"spending_long_harmonized", "revenue_long_harmonized",
|
||||
"spending_annotated_harmonized", "revenue_annotated_harmonized",
|
||||
"harmonization_map", "harmonization_recipes", "series_breaks_pq"
|
||||
)
|
||||
expect_true(all(expected_v5 %in% views$table_name))
|
||||
})
|
||||
|
||||
test_that("spending_long filters to E/F/G/K prefixes and excludes aggregates", {
|
||||
skip_if_no_corpus()
|
||||
con <- cog_open()
|
||||
@@ -69,13 +125,15 @@ test_that("gov_population_yearly exposes one row per (year, canonical_govid)", {
|
||||
WHERE canonical_govid = '121011212191'
|
||||
ORDER BY year"
|
||||
)
|
||||
expect_setequal(df$year, c(2019L, 2020L))
|
||||
expect_equal(nrow(df), 2L)
|
||||
expect_setequal(df$year, c(2011L, 2012L, 2019L, 2020L))
|
||||
expect_equal(nrow(df), 4L)
|
||||
expect_true(all(!is.na(df$population)))
|
||||
# Hardcoded values are from the bundled fixture (regenerated 2026-07-11
|
||||
# against cog_pipeline publish tree, pipeline_commit 1a00925, Phase P
|
||||
# schema_version 4). Update if the fixture is rebuilt against a
|
||||
# different source vintage.
|
||||
# Hardcoded values are from the bundled fixture (regenerated 2026-07-18
|
||||
# against cog_pipeline publish tree, pipeline_commit ece9b32, Phase R2
|
||||
# schema_version 5, years 2011/2012/2019/2020). Update if the fixture is
|
||||
# rebuilt against a different source vintage.
|
||||
expect_equal(df$population[df$year == 2011L], 1759591L)
|
||||
expect_equal(df$population[df$year == 2012L], 1819773L)
|
||||
expect_equal(df$population[df$year == 2019L], 1935878L)
|
||||
expect_equal(df$population[df$year == 2020L], 1952778L)
|
||||
# Uniqueness on (year, canonical_govid) across the whole view.
|
||||
|
||||
Reference in New Issue
Block a user