feat: basis= harmonized/raw with v4/v5 dual-accept

Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.

Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.

Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
2026-07-18 23:19:17 -04:00
parent 3b725770d2
commit 7818cd2b1a
35 changed files with 579 additions and 53 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+42 -13
View File
@@ -1,8 +1,8 @@
{
"schema_version": 4,
"built_at": "2026-07-13T23:35:09Z",
"pipeline_commit": "a082b26",
"fixture_note": "Two-year (2019-2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL. Regenerated for Phase P (schema_version 4, uniformly 12-char canonical_govid) with the full canonical_fips_xwalk master and the new canonical_alias lookup table via data-raw/regenerate_fixture_corpus.R.",
"schema_version": 5,
"built_at": "2026-07-19T03:08:41Z",
"pipeline_commit": "ece9b32",
"fixture_note": "Four-year (2011, 2012, 2019, 2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL. Regenerated for Phase R2 (schema_version 5, harmonization_map/harmonization_recipes/ series_breaks parquet tables added). 2011/2012 straddle the wide-aggregate -> modern-leaf format boundary exercised by basis= \"harmonized\" and recipe= queries; 2019/2020 retain the prior per-capita/CPI regression anchors. Full canonical_fips_xwalk master and canonical_alias lookup table included via data-raw/regenerate_fixture_corpus.R.",
"data_vintage": {
"census_source_downloaded": "unknown",
"cpi_vintage": "FRED CPIAUCSL",
@@ -14,42 +14,71 @@
"scope_note": "v0.1 covers state, county, city/municipality, and township governments. Special districts (type 4) and school districts (type 5) are excluded pending validation in a future cycle."
},
"schema": {
"long_column_count": 24,
"long_columns": ["fips_state", "type", "fips_county", "govid", "gov_blank", "gov_name", "county_name", "fips_state_code", "fips_county_code", "fips_place_code", "population", "popyear", "enrollment", "enrollyear", "function_code", "sch_level_code", "fiscal_year_end", "srvy_year", "item_code", "amt", "srv_data", "impute_flag", "is_aggregate", "canonical_govid"],
"long_column_count": 26,
"long_columns": ["fips_state", "type", "fips_county", "govid", "gov_blank", "gov_name", "county_name", "fips_state_code", "fips_county_code", "fips_place_code", "population", "popyear", "enrollment", "enrollyear", "function_code", "sch_level_code", "fiscal_year_end", "srvy_year", "item_code", "amt", "srv_data", "impute_flag", "is_aggregate", "canonical_govid", "harmonized_code", "survey_weight"],
"data_dictionary": "docs/data_dictionary.md"
},
"files": {
"long_partitions": [
{
"year": 2011,
"path": "data/long/year=2011/part-0.parquet",
"sha256": "76c2153ef0a94c3551a24751aa08225fc937fafb579f10fd0750848d781dc471",
"row_count": 3037606,
"size_bytes": 4985667
},
{
"year": 2012,
"path": "data/long/year=2012/part-0.parquet",
"sha256": "a9bb10d04887490376a4b030a03a4ca1e0f1f9fbca5f7d99ea7ca7434c42b2bc",
"row_count": 1163338,
"size_bytes": 5900191
},
{
"year": 2019,
"path": "data/long/year=2019/part-0.parquet",
"sha256": "c0a2bf0758af129d5dfddb6ff6665cc435ddee87fd6879e788fb56ed53ab22b8",
"sha256": "475481fcfbb030f96cae457cbda824512a2da6f97f4c6d12a75130979e4331f9",
"row_count": 318139,
"size_bytes": 1441404
"size_bytes": 1717345
},
{
"year": 2020,
"path": "data/long/year=2020/part-0.parquet",
"sha256": "92570b9d55ec3425d034db37838f91c3b8359d0454d3d98730a6016b62e4bb48",
"sha256": "7cdf3eb1a73c34e6a3d22befdf7ccdd0b44b1e6b1cf4491159a38a5a24a1b005",
"row_count": 317500,
"size_bytes": 1444011
"size_bytes": 1720794
}
],
"metadata": [
{
"path": "data/canonical_alias.parquet",
"sha256": "feb8d01a640fb16c9a4b4ad66726b50b8fe8a1ce2a771bed8c1190fec51d5c8d",
"sha256": "fa27ce4b59e286ace05fbcb9f59a910b8e483fd6c58ddf01875a210b966809cb",
"description": "canonical_alias.parquet"
},
{
"path": "data/canonical_fips_xwalk.parquet",
"sha256": "1ae47981531c7389f69eff3f7656045428564bfbe8032200eb9c039d32739a7e",
"sha256": "b6e2c4cb748141f6f2e324d6704c13830d2a68ad57bbb73d04ac54b881d1285b",
"description": "canonical_fips_xwalk.parquet"
},
{
"path": "data/summary_categories.parquet",
"sha256": "dd59e7f58a022679ad43511c8c8e938b8dd4be81196bbeeee21e67bdcca2295b",
"sha256": "8e6fcd4dd9bb4723841a67233b19388c9762dfc23b4479501183cebf7ea3c1b5",
"description": "summary_categories.parquet"
},
{
"path": "data/harmonization_map.parquet",
"sha256": "52b4f3f94e65231aa869445946df7a0cfebf8fef1940f800e08287b53d09ab42",
"description": "harmonization_map.parquet"
},
{
"path": "data/harmonization_recipes.parquet",
"sha256": "1133e9a0b02f8f34f5f936e55c5ecd596bb8a55d8425dcce76767f0f3203581c",
"description": "harmonization_recipes.parquet"
},
{
"path": "data/series_breaks.parquet",
"sha256": "049bd7365dae14e357f6e57765dc5a069440bab46facce237d7ba2ec94b0b113",
"description": "series_breaks.parquet"
}
]
},
+5
View File
@@ -10,6 +10,11 @@
"target": { "type": "object" },
"years": { "type": "array", "items": { "type": "integer" } },
"category": { "type": ["string", "array", "null"] },
"basis": { "type": ["string", "null"] },
"basis_note": { "type": ["string", "null"] },
"harmonization": { "type": "object" },
"recipe": { "type": ["object", "null"] },
"suggestions": { "type": "array" },
"scope": { "type": "object" },
"codes_summed": { "type": "object" },
"aggregate_fallback": { "type": ["object", "null"] },
+6
View File
@@ -0,0 +1,6 @@
CREATE OR REPLACE VIEW spending_long_harmonized AS
SELECT * REPLACE (harmonized_code AS item_code)
FROM long
WHERE NOT is_aggregate
AND harmonized_code IS NOT NULL
AND LEFT(harmonized_code, 1) IN ('E', 'F', 'G', 'K');
+6
View File
@@ -0,0 +1,6 @@
CREATE OR REPLACE VIEW revenue_long_harmonized AS
SELECT * REPLACE (harmonized_code AS item_code)
FROM long
WHERE NOT is_aggregate
AND harmonized_code IS NOT NULL
AND LEFT(harmonized_code, 1) IN ('T', 'A', 'U', 'B', 'C', 'D');
+3
View File
@@ -0,0 +1,3 @@
CREATE OR REPLACE VIEW harmonization_map AS
SELECT *
FROM read_parquet('{url}data/harmonization_map.parquet');
+3
View File
@@ -0,0 +1,3 @@
CREATE OR REPLACE VIEW harmonization_recipes AS
SELECT *
FROM read_parquet('{url}data/harmonization_recipes.parquet');
+3
View File
@@ -0,0 +1,3 @@
CREATE OR REPLACE VIEW series_breaks_pq AS
SELECT *
FROM read_parquet('{url}data/series_breaks.parquet');
@@ -0,0 +1,16 @@
CREATE OR REPLACE VIEW spending_annotated_harmonized AS
SELECT
s.*,
x.gov_name AS xwalk_gov_name,
x.govs_type,
x.type_label,
x.fips_state AS xwalk_fips_state,
x.fips_county AS xwalk_fips_county,
x.fips_place,
x.population_acs,
c.category,
c.category_type,
c.spend_subtype
FROM spending_long_harmonized s
LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
LEFT JOIN summary_categories c USING (item_code);
@@ -0,0 +1,16 @@
CREATE OR REPLACE VIEW revenue_annotated_harmonized AS
SELECT
s.*,
x.gov_name AS xwalk_gov_name,
x.govs_type,
x.type_label,
x.fips_state AS xwalk_fips_state,
x.fips_county AS xwalk_fips_county,
x.fips_place,
x.population_acs,
c.category,
c.category_type,
c.revenue_subtype
FROM revenue_long_harmonized s
LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
LEFT JOIN summary_categories c USING (item_code);