feat: basis= harmonized/raw with v4/v5 dual-accept

Adds schema_version 5 support alongside the existing v4 corpus: .validate_schema() now accepts a supported set (4, 5) instead of a single expected version, and cog_spending()/cog_revenue() gain basis = c("harmonized", "raw"). Harmonized basis routes to new spending_annotated_harmonized / revenue_annotated_harmonized views built on spending_long_harmonized / revenue_long_harmonized (REPLACE(harmonized_code AS item_code), excluding aggregate and NA-harmonized rows); raw basis is byte-identical to the pre-Phase-R2 behavior. On a v4 corpus, an unspecified basis silently resolves to "raw" with a provenance note; an explicit basis = "harmonized" aborts with an actionable message. Provenance gains basis, basis_note, and a harmonization block (applied/na_rows_excluded/na_amount_excluded). The five new schema-v5-only SQL views (harmonized long/annotated views, harmonization_map, harmonization_recipes, series_breaks_pq) are registered conditionally on manifest$schema_version >= 5, since DuckDB's read_parquet() errors eagerly at CREATE VIEW time when the backing file doesn't exist on a v4 corpus. Fixture corpus regenerated to schema_version 5 / years 2011, 2012, 2019, 2020 (2011->2012 spans the wide-aggregate -> modern-leaf format boundary needed for the harmonization/recipe work), with the harmonization_map / harmonization_recipes / series_breaks parquet tables bundled alongside the existing metadata registries.
2026-07-18 23:19:17 -04:00
parent 3b725770d2
commit 7818cd2b1a
35 changed files with 579 additions and 53 deletions
@@ -0,0 +1,78 @@
+# R/basis.R
+# basis= resolution (harmonized/raw, with v4/v5 dual-accept) and the
+# harmonization exclusion-count block attached to provenance.
+
+#' Resolve the requested `basis` against the active corpus's schema_version.
+#'
+#' On a `schema_version >= 5` corpus, the requested basis is used as-is. On
+#' an older (`schema_version == 4`) corpus, which has no harmonization
+#' tables: a caller who left `basis` at its default (`"harmonized"`, so
+#' `explicit` is `FALSE`) silently gets `"raw"` back, with a note recorded
+#' for provenance; a caller who explicitly asked for
+#' `basis = "harmonized"` gets a hard abort instead of a silent downgrade.
+#'
+#' @param basis `"harmonized"` or `"raw"` (already resolved via `match.arg`).
+#' @param explicit `TRUE` if the caller passed `basis` explicitly (as
+#'   opposed to relying on the default `c("harmonized", "raw")`).
+#' @param manifest The active session's parsed manifest list.
+#' @return List with `basis` (the resolved value) and `note` (character or
+#'   `NA_character_`).
+#' @noRd
+.resolve_basis <- function(basis, explicit, manifest) {
+  schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
+
+  if (schema_version >= 5L) {
+    return(list(basis = basis, note = NA_character_))
+  }
+
+  if (identical(basis, "harmonized") && explicit) {
+    cli::cli_abort(c(
+      "basis = \"harmonized\" requires corpus schema_version >= 5.",
+      x = "Active corpus has schema_version {schema_version}.",
+      i = "Use basis = \"raw\" (the default on this corpus), or point USCOGDATA_URL at a schema_version >= 5 corpus."
+    ), class = "uscogdata_basis_unsupported")
+  }
+
+  list(
+    basis = "raw",
+    note = sprintf(
+      "basis resolved to \"raw\": corpus schema_version %d < 5 (harmonization tables unavailable)",
+      schema_version
+    )
+  )
+}
+
+#' Count + sum item-level rows that basis="harmonized" excludes because they
+#' carry no harmonized_code (discontinued / not-yet-ruled codes) within the
+#' requested flow type (spending or revenue), govids, and years. Only
+#' meaningful when the resolved basis is "harmonized"; returns an
+#' applied = FALSE stub otherwise (raw basis never excludes rows this way).
+#' @noRd
+.build_harmonization_block <- function(con, govid, years, resolved, flow_prefixes) {
+  if (!identical(resolved$basis, "harmonized")) {
+    return(list(
+      applied = FALSE,
+      na_rows_excluded = 0L,
+      na_amount_excluded = 0,
+      note = resolved$note
+    ))
+  }
+
+  sql <- sprintf(
+    "SELECT COUNT(*) AS n, COALESCE(SUM(amt), 0) * 1000.0 AS amt
+     FROM long
+     WHERE canonical_govid IN (%s) AND year IN (%s)
+       AND NOT is_aggregate AND harmonized_code IS NULL
+       AND LEFT(item_code, 1) IN (%s)",
+    .sql_lit_chr(govid), paste(as.integer(years), collapse = ","),
+    .sql_lit_chr(flow_prefixes)
+  )
+  na <- DBI::dbGetQuery(con, sql)
+
+  list(
+    applied = TRUE,
+    na_rows_excluded = as.integer(na$n),
+    na_amount_excluded = as.numeric(na$amt),
+    note = resolved$note
+  )
+}