Civilytics/uscogdata
3
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: basis= harmonized/raw with v4/v5 dual-accept

Browse Source 
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.

Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.

Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
This commit is contained in:
jared
2026-07-18 23:19:17 -04:00
parent 3b725770d2
commit 7818cd2b1a
35 changed files with 579 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
R/basis.R
+78
View File
@@ -0,0 +1,78 @@
# R/basis.R
# basis= resolution (harmonized/raw, with v4/v5 dual-accept) and the
# harmonization exclusion-count block attached to provenance.
#' Resolve the requested `basis` against the active corpus's schema_version.
#'
#' On a `schema_version >= 5` corpus, the requested basis is used as-is. On
#' an older (`schema_version == 4`) corpus, which has no harmonization
#' tables: a caller who left `basis` at its default (`"harmonized"`, so
#' `explicit` is `FALSE`) silently gets `"raw"` back, with a note recorded
#' for provenance; a caller who explicitly asked for
#' `basis = "harmonized"` gets a hard abort instead of a silent downgrade.
#'
#' @param basis `"harmonized"` or `"raw"` (already resolved via `match.arg`).
#' @param explicit `TRUE` if the caller passed `basis` explicitly (as
#' opposed to relying on the default `c("harmonized", "raw")`).
#' @param manifest The active session's parsed manifest list.
#' @return List with `basis` (the resolved value) and `note` (character or
#' `NA_character_`).
#' @noRd
.resolve_basis <- function(basis, explicit, manifest) {
schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
if (schema_version >= 5L) {
return(list(basis = basis, note = NA_character_))
}
if (identical(basis, "harmonized") && explicit) {
cli::cli_abort(c(
"basis = \"harmonized\" requires corpus schema_version >= 5.",
x = "Active corpus has schema_version {schema_version}.",
i = "Use basis = \"raw\" (the default on this corpus), or point USCOGDATA_URL at a schema_version >= 5 corpus."
), class = "uscogdata_basis_unsupported")
}
list(
basis = "raw",
note = sprintf(
"basis resolved to \"raw\": corpus schema_version %d < 5 (harmonization tables unavailable)",
schema_version
)
)
}
#' Count + sum item-level rows that basis="harmonized" excludes because they
#' carry no harmonized_code (discontinued / not-yet-ruled codes) within the
#' requested flow type (spending or revenue), govids, and years. Only
#' meaningful when the resolved basis is "harmonized"; returns an
#' applied = FALSE stub otherwise (raw basis never excludes rows this way).
#' @noRd
.build_harmonization_block <- function(con, govid, years, resolved, flow_prefixes) {
if (!identical(resolved$basis, "harmonized")) {
return(list(
applied = FALSE,
na_rows_excluded = 0L,
na_amount_excluded = 0,
note = resolved$note
))
}
sql <- sprintf(
"SELECT COUNT(*) AS n, COALESCE(SUM(amt), 0) * 1000.0 AS amt
FROM long
WHERE canonical_govid IN (%s) AND year IN (%s)
AND NOT is_aggregate AND harmonized_code IS NULL
AND LEFT(item_code, 1) IN (%s)",
.sql_lit_chr(govid), paste(as.integer(years), collapse = ","),
.sql_lit_chr(flow_prefixes)
)
na <- DBI::dbGetQuery(con, sql)
list(
applied = TRUE,
na_rows_excluded = as.integer(na$n),
na_amount_excluded = as.numeric(na$amt),
note = resolved$note
)
}
R/manifest.R
+3 -3
View File
@@ -128,11 +128,11 @@
}
#' @noRd
.validate_schema <- function(manifest, expected_version) {
if (manifest$schema_version != expected_version) {
.validate_schema <- function(manifest, supported = c(4L, 5L)) {
if (!manifest$schema_version %in% supported) {
cli::cli_abort(c(
"Corpus schema version mismatch.",
x = "Package expects schema_version = {expected_version}; corpus has {manifest$schema_version}.",
x = "Package supports schema_version in {paste(supported, collapse = ', ')}; corpus has {manifest$schema_version}.",
i = "Update uscogdata (install.packages or pak::pkg_install) or re-publish corpus."
))
}
R/provenance.R
+9 -1
View File
@@ -4,7 +4,9 @@
#' @noRd
.build_provenance <- function(verb, call, govid, years, category,
per_capita, adjust_to_year, result, sql,
subtype_col) {
subtype_col, basis = NA_character_,
basis_note = NA_character_,
harmonization = NULL) {
manifest <- .uscogdata_env$manifest
codes <- result[["codes_included"]]
@@ -38,6 +40,12 @@
),
years = as.integer(years),
category = category,
basis = basis,
basis_note = basis_note,
harmonization = harmonization %||% list(
applied = FALSE, na_rows_excluded = 0L, na_amount_excluded = 0,
note = NA_character_
),
scope = list(
gov_types_included = as.integer(unlist(manifest$scope$gov_types_included)),
gov_types_excluded = as.integer(unlist(manifest$scope$gov_types_excluded)),
R/revenue.R
+6 -3
View File
@@ -14,16 +14,19 @@
#' optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`.
#' @export
cog_revenue <- function(govid, years, category = NULL,
per_capita = FALSE, adjust_to_year = NULL) {
per_capita = FALSE, adjust_to_year = NULL,
basis = c("harmonized", "raw")) {
.verb_spendrev(
verb = "cog_revenue",
view = "revenue_annotated",
view_base = "revenue_annotated",
subtype_col = "revenue_subtype",
flow_prefixes = c("T", "A", "U", "B", "C", "D"),
call = match.call(),
govid = govid,
years = years,
category = category,
per_capita = per_capita,
adjust_to_year = adjust_to_year
adjust_to_year = adjust_to_year,
basis = basis
)
}
R/session.R
+1 -1
View File
@@ -12,7 +12,7 @@ cog_open <- function(url = .resolve_url(),
DBI::dbExecute(con, "INSTALL httpfs; LOAD httpfs;")
manifest <- .fetch_or_cache_manifest(url, cache_dir)
.validate_schema(manifest, expected_version = 4L)
.validate_schema(manifest, supported = c(4L, 5L))
.validate_scope(manifest)
.register_views(con, url, manifest)
R/spending.R
+38 -6
View File
@@ -20,6 +20,15 @@
#' in that year).
#' @param adjust_to_year Integer base year for CPI-U real-dollar conversion,
#' or `NULL` for nominal only.
#' @param basis `"harmonized"` (default) sums item codes through the
#' cross-vintage harmonization mapping (folding series-break-affected
#' codes onto a comparable target and excluding aggregate / discontinued
#' rows -- see the `harmonization` block in `cog_explain()`); `"raw"`
#' reproduces the pre-Phase-R2 behavior (published item codes, no
#' folding). On a corpus with `schema_version < 5` (no harmonization
#' tables), `basis` silently resolves to `"raw"` when left at its default
#' and the resolution is recorded in the provenance; explicitly passing
#' `basis = "harmonized"` on such a corpus aborts.
#' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
#' `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`,
#' optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
@@ -27,24 +36,31 @@
#' Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`.
#' @export
cog_spending <- function(govid, years, category = NULL,
per_capita = FALSE, adjust_to_year = NULL) {
per_capita = FALSE, adjust_to_year = NULL,
basis = c("harmonized", "raw")) {
.verb_spendrev(
verb = "cog_spending",
view = "spending_annotated",
view_base = "spending_annotated",
subtype_col = "spend_subtype",
flow_prefixes = c("E", "F", "G", "K"),
call = match.call(),
govid = govid,
years = years,
category = category,
per_capita = per_capita,
adjust_to_year = adjust_to_year
adjust_to_year = adjust_to_year,
basis = basis
)
}
#' @noRd
.verb_spendrev <- function(verb, view, subtype_col, call,
.verb_spendrev <- function(verb, view_base, subtype_col, flow_prefixes, call,
govid, years, category,
per_capita, adjust_to_year) {
per_capita, adjust_to_year,
basis = c("harmonized", "raw")) {
basis_explicit <- length(basis) == 1L
basis <- match.arg(basis, c("harmonized", "raw"))
govid <- .coerce_govid_input(govid, arg = "govid")
.validate_verb_inputs(govid, years, category, per_capita, adjust_to_year)
@@ -52,8 +68,12 @@ cog_spending <- function(govid, years, category = NULL,
if (!is.null(adjust_to_year)) adjust_to_year <- as.integer(adjust_to_year)
con <- .ensure_session()
manifest <- .uscogdata_env$manifest
scope <- .check_govids_in_scope(govid)
resolved <- .resolve_basis(basis, basis_explicit, manifest)
view <- .select_view(view_base, resolved$basis)
sql <- .build_verb_sql(view, subtype_col, govid, years, category)
result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
@@ -64,6 +84,10 @@ cog_spending <- function(govid, years, category = NULL,
result$notes <- .notes_column(result)
harmonization <- .build_harmonization_block(
con, govid, years, resolved, flow_prefixes
)
prov <- .build_provenance(
verb = verb,
call = call,
@@ -74,7 +98,10 @@ cog_spending <- function(govid, years, category = NULL,
adjust_to_year = adjust_to_year,
result = result,
sql = sql,
subtype_col = subtype_col
subtype_col = subtype_col,
basis = resolved$basis,
basis_note = resolved$note,
harmonization = harmonization
)
prov$scope$govids_found <- scope$found
prov$scope$govids_missing <- scope$missing
@@ -107,6 +134,11 @@ cog_spending <- function(govid, years, category = NULL,
invisible(TRUE)
}
#' @noRd
.select_view <- function(view_base, basis) {
if (identical(basis, "harmonized")) paste0(view_base, "_harmonized") else view_base
}
#' @noRd
.sql_lit_chr <- function(x) {
safe <- gsub("'", "''", x, fixed = TRUE)
R/views.R
+23 -1
View File
@@ -1,11 +1,33 @@
# R/views.R
# SQL files whose view definitions read schema-v5-only parquet tables
# (harmonization_map.parquet, harmonization_recipes.parquet,
# series_breaks.parquet) or select from views built on top of them. DuckDB's
# read_parquet() resolves the file at CREATE VIEW time (even for a view, it
# still needs the source schema) and errors immediately -- "IO Error: No
# files found" -- if the path doesn't exist, so these cannot be registered
# unconditionally against a v4 corpus the way the rest of inst/sql/ is.
# Registration is therefore gated on manifest$schema_version >= 5; verb-level
# *usage* of the resulting views is separately gated by .resolve_basis() /
# .require_schema_v5().
.harmonization_view_files <- c(
"22-spending_long_harmonized.sql",
"23-revenue_long_harmonized.sql",
"33-harmonization_map.sql",
"34-harmonization_recipes.sql",
"35-series_breaks_pq.sql",
"42-spending_annotated_harmonized.sql",
"43-revenue_annotated_harmonized.sql"
)
#' Register DuckDB views from inst/sql/ SQL files
#' @noRd
.register_views <- function(con, url, manifest) {
sql_dir <- system.file("sql", package = "uscogdata")
files <- list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE)
files <- sort(list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE))
schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
for (f in files) {
if (basename(f) %in% .harmonization_view_files && schema_version < 5L) next
sql <- paste(readLines(f, warn = FALSE), collapse = "\n")
sql <- gsub("\\{url\\}", url, sql, fixed = FALSE)
DBI::dbExecute(con, sql)