jared
7818cd2b1a
Adds schema_version 5 support alongside the existing v4 corpus:
.validate_schema() now accepts a supported set (4, 5) instead of a single
expected version, and cog_spending()/cog_revenue() gain basis =
c("harmonized", "raw"). Harmonized basis routes to new
spending_annotated_harmonized / revenue_annotated_harmonized views built on
spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code
AS item_code), excluding aggregate and NA-harmonized rows); raw basis is
byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified
basis silently resolves to "raw" with a provenance note; an explicit
basis = "harmonized" aborts with an actionable message.
Provenance gains basis, basis_note, and a harmonization block
(applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only
SQL views (harmonized long/annotated views, harmonization_map,
harmonization_recipes, series_breaks_pq) are registered conditionally on
manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly
at CREATE VIEW time when the backing file doesn't exist on a v4 corpus.
Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019,
2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary
needed for the harmonization/recipe work), with the harmonization_map /
harmonization_recipes / series_breaks parquet tables bundled alongside the
existing metadata registries.
79 lines
2.9 KiB
R
79 lines
2.9 KiB
R
# R/basis.R
|
|
# basis= resolution (harmonized/raw, with v4/v5 dual-accept) and the
|
|
# harmonization exclusion-count block attached to provenance.
|
|
|
|
#' Resolve the requested `basis` against the active corpus's schema_version.
|
|
#'
|
|
#' On a `schema_version >= 5` corpus, the requested basis is used as-is. On
|
|
#' an older (`schema_version == 4`) corpus, which has no harmonization
|
|
#' tables: a caller who left `basis` at its default (`"harmonized"`, so
|
|
#' `explicit` is `FALSE`) silently gets `"raw"` back, with a note recorded
|
|
#' for provenance; a caller who explicitly asked for
|
|
#' `basis = "harmonized"` gets a hard abort instead of a silent downgrade.
|
|
#'
|
|
#' @param basis `"harmonized"` or `"raw"` (already resolved via `match.arg`).
|
|
#' @param explicit `TRUE` if the caller passed `basis` explicitly (as
|
|
#' opposed to relying on the default `c("harmonized", "raw")`).
|
|
#' @param manifest The active session's parsed manifest list.
|
|
#' @return List with `basis` (the resolved value) and `note` (character or
|
|
#' `NA_character_`).
|
|
#' @noRd
|
|
.resolve_basis <- function(basis, explicit, manifest) {
|
|
schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
|
|
|
|
if (schema_version >= 5L) {
|
|
return(list(basis = basis, note = NA_character_))
|
|
}
|
|
|
|
if (identical(basis, "harmonized") && explicit) {
|
|
cli::cli_abort(c(
|
|
"basis = \"harmonized\" requires corpus schema_version >= 5.",
|
|
x = "Active corpus has schema_version {schema_version}.",
|
|
i = "Use basis = \"raw\" (the default on this corpus), or point USCOGDATA_URL at a schema_version >= 5 corpus."
|
|
), class = "uscogdata_basis_unsupported")
|
|
}
|
|
|
|
list(
|
|
basis = "raw",
|
|
note = sprintf(
|
|
"basis resolved to \"raw\": corpus schema_version %d < 5 (harmonization tables unavailable)",
|
|
schema_version
|
|
)
|
|
)
|
|
}
|
|
|
|
#' Count + sum item-level rows that basis="harmonized" excludes because they
|
|
#' carry no harmonized_code (discontinued / not-yet-ruled codes) within the
|
|
#' requested flow type (spending or revenue), govids, and years. Only
|
|
#' meaningful when the resolved basis is "harmonized"; returns an
|
|
#' applied = FALSE stub otherwise (raw basis never excludes rows this way).
|
|
#' @noRd
|
|
.build_harmonization_block <- function(con, govid, years, resolved, flow_prefixes) {
|
|
if (!identical(resolved$basis, "harmonized")) {
|
|
return(list(
|
|
applied = FALSE,
|
|
na_rows_excluded = 0L,
|
|
na_amount_excluded = 0,
|
|
note = resolved$note
|
|
))
|
|
}
|
|
|
|
sql <- sprintf(
|
|
"SELECT COUNT(*) AS n, COALESCE(SUM(amt), 0) * 1000.0 AS amt
|
|
FROM long
|
|
WHERE canonical_govid IN (%s) AND year IN (%s)
|
|
AND NOT is_aggregate AND harmonized_code IS NULL
|
|
AND LEFT(item_code, 1) IN (%s)",
|
|
.sql_lit_chr(govid), paste(as.integer(years), collapse = ","),
|
|
.sql_lit_chr(flow_prefixes)
|
|
)
|
|
na <- DBI::dbGetQuery(con, sql)
|
|
|
|
list(
|
|
applied = TRUE,
|
|
na_rows_excluded = as.integer(na$n),
|
|
na_amount_excluded = as.numeric(na$amt),
|
|
note = resolved$note
|
|
)
|
|
}
|