feat: basis= harmonized/raw with v4/v5 dual-accept
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.
Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.
Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+42
-13
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"schema_version": 4,
|
||||
"built_at": "2026-07-13T23:35:09Z",
|
||||
"pipeline_commit": "a082b26",
|
||||
"fixture_note": "Two-year (2019-2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL. Regenerated for Phase P (schema_version 4, uniformly 12-char canonical_govid) with the full canonical_fips_xwalk master and the new canonical_alias lookup table via data-raw/regenerate_fixture_corpus.R.",
|
||||
"schema_version": 5,
|
||||
"built_at": "2026-07-19T03:08:41Z",
|
||||
"pipeline_commit": "ece9b32",
|
||||
"fixture_note": "Four-year (2011, 2012, 2019, 2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL. Regenerated for Phase R2 (schema_version 5, harmonization_map/harmonization_recipes/ series_breaks parquet tables added). 2011/2012 straddle the wide-aggregate -> modern-leaf format boundary exercised by basis= \"harmonized\" and recipe= queries; 2019/2020 retain the prior per-capita/CPI regression anchors. Full canonical_fips_xwalk master and canonical_alias lookup table included via data-raw/regenerate_fixture_corpus.R.",
|
||||
"data_vintage": {
|
||||
"census_source_downloaded": "unknown",
|
||||
"cpi_vintage": "FRED CPIAUCSL",
|
||||
@@ -14,42 +14,71 @@
|
||||
"scope_note": "v0.1 covers state, county, city/municipality, and township governments. Special districts (type 4) and school districts (type 5) are excluded pending validation in a future cycle."
|
||||
},
|
||||
"schema": {
|
||||
"long_column_count": 24,
|
||||
"long_columns": ["fips_state", "type", "fips_county", "govid", "gov_blank", "gov_name", "county_name", "fips_state_code", "fips_county_code", "fips_place_code", "population", "popyear", "enrollment", "enrollyear", "function_code", "sch_level_code", "fiscal_year_end", "srvy_year", "item_code", "amt", "srv_data", "impute_flag", "is_aggregate", "canonical_govid"],
|
||||
"long_column_count": 26,
|
||||
"long_columns": ["fips_state", "type", "fips_county", "govid", "gov_blank", "gov_name", "county_name", "fips_state_code", "fips_county_code", "fips_place_code", "population", "popyear", "enrollment", "enrollyear", "function_code", "sch_level_code", "fiscal_year_end", "srvy_year", "item_code", "amt", "srv_data", "impute_flag", "is_aggregate", "canonical_govid", "harmonized_code", "survey_weight"],
|
||||
"data_dictionary": "docs/data_dictionary.md"
|
||||
},
|
||||
"files": {
|
||||
"long_partitions": [
|
||||
{
|
||||
"year": 2011,
|
||||
"path": "data/long/year=2011/part-0.parquet",
|
||||
"sha256": "76c2153ef0a94c3551a24751aa08225fc937fafb579f10fd0750848d781dc471",
|
||||
"row_count": 3037606,
|
||||
"size_bytes": 4985667
|
||||
},
|
||||
{
|
||||
"year": 2012,
|
||||
"path": "data/long/year=2012/part-0.parquet",
|
||||
"sha256": "a9bb10d04887490376a4b030a03a4ca1e0f1f9fbca5f7d99ea7ca7434c42b2bc",
|
||||
"row_count": 1163338,
|
||||
"size_bytes": 5900191
|
||||
},
|
||||
{
|
||||
"year": 2019,
|
||||
"path": "data/long/year=2019/part-0.parquet",
|
||||
"sha256": "c0a2bf0758af129d5dfddb6ff6665cc435ddee87fd6879e788fb56ed53ab22b8",
|
||||
"sha256": "475481fcfbb030f96cae457cbda824512a2da6f97f4c6d12a75130979e4331f9",
|
||||
"row_count": 318139,
|
||||
"size_bytes": 1441404
|
||||
"size_bytes": 1717345
|
||||
},
|
||||
{
|
||||
"year": 2020,
|
||||
"path": "data/long/year=2020/part-0.parquet",
|
||||
"sha256": "92570b9d55ec3425d034db37838f91c3b8359d0454d3d98730a6016b62e4bb48",
|
||||
"sha256": "7cdf3eb1a73c34e6a3d22befdf7ccdd0b44b1e6b1cf4491159a38a5a24a1b005",
|
||||
"row_count": 317500,
|
||||
"size_bytes": 1444011
|
||||
"size_bytes": 1720794
|
||||
}
|
||||
],
|
||||
"metadata": [
|
||||
{
|
||||
"path": "data/canonical_alias.parquet",
|
||||
"sha256": "feb8d01a640fb16c9a4b4ad66726b50b8fe8a1ce2a771bed8c1190fec51d5c8d",
|
||||
"sha256": "fa27ce4b59e286ace05fbcb9f59a910b8e483fd6c58ddf01875a210b966809cb",
|
||||
"description": "canonical_alias.parquet"
|
||||
},
|
||||
{
|
||||
"path": "data/canonical_fips_xwalk.parquet",
|
||||
"sha256": "1ae47981531c7389f69eff3f7656045428564bfbe8032200eb9c039d32739a7e",
|
||||
"sha256": "b6e2c4cb748141f6f2e324d6704c13830d2a68ad57bbb73d04ac54b881d1285b",
|
||||
"description": "canonical_fips_xwalk.parquet"
|
||||
},
|
||||
{
|
||||
"path": "data/summary_categories.parquet",
|
||||
"sha256": "dd59e7f58a022679ad43511c8c8e938b8dd4be81196bbeeee21e67bdcca2295b",
|
||||
"sha256": "8e6fcd4dd9bb4723841a67233b19388c9762dfc23b4479501183cebf7ea3c1b5",
|
||||
"description": "summary_categories.parquet"
|
||||
},
|
||||
{
|
||||
"path": "data/harmonization_map.parquet",
|
||||
"sha256": "52b4f3f94e65231aa869445946df7a0cfebf8fef1940f800e08287b53d09ab42",
|
||||
"description": "harmonization_map.parquet"
|
||||
},
|
||||
{
|
||||
"path": "data/harmonization_recipes.parquet",
|
||||
"sha256": "1133e9a0b02f8f34f5f936e55c5ecd596bb8a55d8425dcce76767f0f3203581c",
|
||||
"description": "harmonization_recipes.parquet"
|
||||
},
|
||||
{
|
||||
"path": "data/series_breaks.parquet",
|
||||
"sha256": "049bd7365dae14e357f6e57765dc5a069440bab46facce237d7ba2ec94b0b113",
|
||||
"description": "series_breaks.parquet"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -10,6 +10,11 @@
|
||||
"target": { "type": "object" },
|
||||
"years": { "type": "array", "items": { "type": "integer" } },
|
||||
"category": { "type": ["string", "array", "null"] },
|
||||
"basis": { "type": ["string", "null"] },
|
||||
"basis_note": { "type": ["string", "null"] },
|
||||
"harmonization": { "type": "object" },
|
||||
"recipe": { "type": ["object", "null"] },
|
||||
"suggestions": { "type": "array" },
|
||||
"scope": { "type": "object" },
|
||||
"codes_summed": { "type": "object" },
|
||||
"aggregate_fallback": { "type": ["object", "null"] },
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE OR REPLACE VIEW spending_long_harmonized AS
|
||||
SELECT * REPLACE (harmonized_code AS item_code)
|
||||
FROM long
|
||||
WHERE NOT is_aggregate
|
||||
AND harmonized_code IS NOT NULL
|
||||
AND LEFT(harmonized_code, 1) IN ('E', 'F', 'G', 'K');
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE OR REPLACE VIEW revenue_long_harmonized AS
|
||||
SELECT * REPLACE (harmonized_code AS item_code)
|
||||
FROM long
|
||||
WHERE NOT is_aggregate
|
||||
AND harmonized_code IS NOT NULL
|
||||
AND LEFT(harmonized_code, 1) IN ('T', 'A', 'U', 'B', 'C', 'D');
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE OR REPLACE VIEW harmonization_map AS
|
||||
SELECT *
|
||||
FROM read_parquet('{url}data/harmonization_map.parquet');
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE OR REPLACE VIEW harmonization_recipes AS
|
||||
SELECT *
|
||||
FROM read_parquet('{url}data/harmonization_recipes.parquet');
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE OR REPLACE VIEW series_breaks_pq AS
|
||||
SELECT *
|
||||
FROM read_parquet('{url}data/series_breaks.parquet');
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE OR REPLACE VIEW spending_annotated_harmonized AS
|
||||
SELECT
|
||||
s.*,
|
||||
x.gov_name AS xwalk_gov_name,
|
||||
x.govs_type,
|
||||
x.type_label,
|
||||
x.fips_state AS xwalk_fips_state,
|
||||
x.fips_county AS xwalk_fips_county,
|
||||
x.fips_place,
|
||||
x.population_acs,
|
||||
c.category,
|
||||
c.category_type,
|
||||
c.spend_subtype
|
||||
FROM spending_long_harmonized s
|
||||
LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
|
||||
LEFT JOIN summary_categories c USING (item_code);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE OR REPLACE VIEW revenue_annotated_harmonized AS
|
||||
SELECT
|
||||
s.*,
|
||||
x.gov_name AS xwalk_gov_name,
|
||||
x.govs_type,
|
||||
x.type_label,
|
||||
x.fips_state AS xwalk_fips_state,
|
||||
x.fips_county AS xwalk_fips_county,
|
||||
x.fips_place,
|
||||
x.population_acs,
|
||||
c.category,
|
||||
c.category_type,
|
||||
c.revenue_subtype
|
||||
FROM revenue_long_harmonized s
|
||||
LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
|
||||
LEFT JOIN summary_categories c USING (item_code);
|
||||
Reference in New Issue
Block a user