R/provenance.R

@@ -8,6 +8,7 @@
                               basis_note = NA_character_,
                               expenditure_concept = "direct",
                               expenditure_concept_note = NA_character_,
+                              expenditure_concept_direct_suppressed = FALSE,
                               harmonization = NULL, recipe = NULL,
                               suggestions = list()) {
   manifest <- .uscogdata_env$manifest
@@ -55,6 +56,7 @@
     basis_note = basis_note,
     expenditure_concept = expenditure_concept,
     expenditure_concept_note = expenditure_concept_note,
+    expenditure_concept_direct_suppressed = isTRUE(expenditure_concept_direct_suppressed),
     harmonization = harmonization %||% list(
       applied = FALSE, na_rows_excluded = 0L, na_amount_excluded = 0,
       note = NA_character_

R/spending.R

@@ -48,13 +48,28 @@
 #'   intergovernmental leg -- payments to local governments (`M` codes) and
 #'   to the state government (`L` codes, excluding the `L--` family-total
 #'   rollup) -- so results gain rows with `spend_subtype ==
-#'   "intergovernmental"`. Mutually exclusive with `recipe` (a recipe
+#'   "intergovernmental"`. Requires the active corpus's `summary_categories`
+#'   to carry M/L rows (added by cog_pipeline PR #59); aborts with class
+#'   `uscogdata_ig_categories_unsupported` on an older corpus rather than
+#'   silently under-reporting. Mutually exclusive with `recipe` (a recipe
 #'   already defines its own component codes). **Do not sum `"total"`
 #'   results across levels of government** (e.g. state + county + city):
 #'   a state's `M12` payment to a school district is the same dollar the
 #'   district reports as its own direct `E12`, so summing both double-counts
 #'   it. This matters in particular with [cog_geographic_rollup()], which
 #'   sums across exactly that kind of multi-layer government set.
+#'
+#'   In the legacy wide era (<= FY2011), some functions are published ONLY
+#'   as an aggregate-flagged family total (e.g. Corrections' `E04`/`E05`
+#'   split), which the Direct leg excludes by construction but the IG leg
+#'   deliberately keeps (see `inst/sql/24-ig_long.sql`). For a `"total"`
+#'   query, any (year, category) where this leaves intergovernmental rows
+#'   with NO Direct counterpart is flagged: the affected rows' `notes`
+#'   name the harmonization recipe that recovers the missing Direct
+#'   component (when one exists), and
+#'   `provenance$expenditure_concept_direct_suppressed` is `TRUE` -- the
+#'   figure in those rows is the intergovernmental leg alone, not Direct +
+#'   IG.
 #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
 #'   `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`,
 #'   optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
@@ -177,6 +192,7 @@ cog_spending <- function(govid, years, category = NULL,
   } else {
     view <- .select_view(view_base, resolved$basis)
     ig_view <- if (identical(expenditure_concept, "total")) {
+      .require_ig_categories(con)
       .select_ig_view(resolved$basis)
     } else {
       NULL
@@ -190,8 +206,6 @@ cog_spending <- function(govid, years, category = NULL,
     result <- .attach_real_dollars(result, adjust_to_year, per_capita)
   }
 
-  result$notes <- .notes_column(result)
-
   # A recipe result doesn't go through spending_annotated(_harmonized) /
   # revenue_annotated(_harmonized) at all -- .run_recipe()'s generic join
   # reads `long` directly -- so `basis` and the `harmonization` exclusion
@@ -217,14 +231,58 @@ cog_spending <- function(govid, years, category = NULL,
     harmonization <- .build_harmonization_block(
       con, govid, years, resolved, flow_prefixes
     )
-    suggestions <- .build_suggestions(con, govid, years, category, result,
+    # C1(a): gap detection must run against the Direct leg alone. `result`
+    # can also carry UNION'd intergovernmental rows (expenditure_concept =
+    # "total"), and the wide era (<= FY2011) routinely has legacy IG dollars
+    # surviving (ig_long deliberately keeps aggregate rows) for a
+    # (year, category) whose legacy Direct dollars were suppressed (spending_
+    # long/spending_long_harmonized both filter NOT is_aggregate). Passing
+    # the UNION'd result here would let a surviving IG row count as coverage
+    # and silently cancel the recipe-hint suggestion that should fire.
+    direct_leg_result <- if (identical(expenditure_concept, "total")) {
+      result[!(result[[subtype_col]] %in% "intergovernmental"), , drop = FALSE]
+    } else {
+      result
+    }
+    suggestions <- .build_suggestions(con, govid, years, category,
+                                       direct_leg_result,
                                        resolved$basis, flow_prefixes)
   }
 
+  # C1(b): when expenditure_concept = "total", flag any row where the IG
+  # leg has dollars but the Direct leg has none for that same (year,
+  # canonical_govid, category) -- the UNION'd figure there is
+  # intergovernmental money ALONE, not Direct + IG, and both the row-level
+  # notes and the provenance must say so rather than pass silently as a
+  # plausible Total.
+  direct_suppressed <- if (identical(expenditure_concept, "total")) {
+    .detect_direct_suppressed(result, subtype_col)
+  } else {
+    rep(FALSE, nrow(result))
+  }
+  direct_suppressed_flag <- isTRUE(any(direct_suppressed))
+
+  result$notes <- .notes_column(result, direct_suppressed, suggestions)
+
   # Determine expenditure_concept_note: only non-empty for "total", explains
-  # how the IG leg was assembled from legacy-era aggregates.
+  # how the IG leg was assembled from legacy-era aggregates. When the Direct
+  # leg is suppressed for at least one requested (year, category), append an
+  # explicit warning rather than let the base note's "Total = Direct + IG"
+  # framing stand unqualified for rows where that arithmetic didn't happen.
   expenditure_concept_note_for_prov <- if (identical(expenditure_concept, "total")) {
-    "Total = Direct + intergovernmental (M to local govts + L to state govts). Legacy-era IG is assembled from aggregate-flagged rows, which are year-disjoint from their modern leaf components; the L-- family total is excluded."
+    base_note <- "Total = Direct + intergovernmental (M to local govts + L to state govts). Legacy-era IG is assembled from aggregate-flagged rows, which are year-disjoint from their modern leaf components; the L-- family total is excluded."
+    if (direct_suppressed_flag) {
+      paste0(
+        base_note,
+        " NOTE: for at least one requested (year, category) the Direct leg ",
+        "has NO rows in this corpus (a legacy aggregate-only family) -- the ",
+        "affected result rows report the intergovernmental leg alone, not ",
+        "Direct + IG. See `expenditure_concept_direct_suppressed` and each ",
+        "affected row's `notes`."
+      )
+    } else {
+      base_note
+    }
   } else {
     NA_character_
   }
@@ -244,6 +302,7 @@ cog_spending <- function(govid, years, category = NULL,
     basis_note     = basis_note_for_prov,
     expenditure_concept = expenditure_concept,
     expenditure_concept_note = expenditure_concept_note_for_prov,
+    expenditure_concept_direct_suppressed = direct_suppressed_flag,
     harmonization  = harmonization,
     recipe         = recipe_block,
     suggestions    = suggestions
@@ -303,6 +362,38 @@ cog_spending <- function(govid, years, category = NULL,
   if (identical(basis, "harmonized")) "ig_annotated_harmonized" else "ig_annotated"
 }
 
+#' Abort unless the active corpus's `summary_categories` actually carries
+#' intergovernmental (M/L) rows.
+#'
+#' The 66 M/L category rows arrived via cog_pipeline PR #59 with NO
+#' `schema_version` bump (`DESCRIPTION` still declares `MinCorpusSchema: 4`),
+#' so `schema_version` alone cannot gate `expenditure_concept = "total"` --
+#' a pre-#59 corpus can validly report schema_version 4, 5, or 6 and still
+#' have zero M/L rows in `summary_categories`. Against such a corpus,
+#' `ig_annotated`'s LEFT JOIN to `summary_categories` silently produces NA
+#' `category`/`spend_subtype` for every IG row: with a `category` filter
+#' this returns 0 rows (reads as "no intergovernmental spending" rather than
+#' "can't tell"), and with `category = NULL` every IG dollar collapses into
+#' one NA-subtype group that is invisible to the `spend_subtype ==
+#' "intergovernmental"` filter this package's own tests, roxygen, and
+#' vignette all rely on. Checking the data directly (rather than
+#' schema_version) is the only reliable gate.
+#' @noRd
+.require_ig_categories <- function(con, what = "expenditure_concept = \"total\"") {
+  n <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS n FROM summary_categories WHERE LEFT(item_code, 1) IN ('M', 'L')"
+  )$n
+  if (identical(as.integer(n), 0L)) {
+    cli::cli_abort(c(
+      sprintf("%s requires a corpus with intergovernmental category rows.", what),
+      x = "The active corpus's `summary_categories` has no M/L (intergovernmental) rows.",
+      i = "This corpus predates the intergovernmental category rows added by cog_pipeline PR #59.",
+      i = "Point USCOGDATA_URL at a newer corpus that includes the M/L summary_categories rows."
+    ), class = "uscogdata_ig_categories_unsupported")
+  }
+  invisible(TRUE)
+}
+
 #' @noRd
 .sql_lit_chr <- function(x) {
   safe <- gsub("'", "''", x, fixed = TRUE)
@@ -409,11 +500,52 @@ cog_spending <- function(govid, years, category = NULL,
   result
 }
 
+#' Detect rows where expenditure_concept = "total" is reporting the
+#' intergovernmental leg with NO Direct counterpart in the same (year,
+#' canonical_govid, category) group -- i.e. the Direct leg is suppressed
+#' (typically a legacy aggregate-only family, see C1(a) above) rather than
+#' genuinely zero. `TRUE` only for the `spend_subtype == "intergovernmental"`
+#' row(s) in each such group.
 #' @noRd
-.notes_column <- function(result) {
+.detect_direct_suppressed <- function(result, subtype_col) {
+  n <- nrow(result)
+  if (n == 0L) return(logical(0))
+  is_ig <- result[[subtype_col]] %in% "intergovernmental"
+  if (!any(is_ig)) return(rep(FALSE, n))
+  key <- paste(result$year, result$canonical_govid, result$category, sep = "\r")
+  has_direct <- key %in% unique(key[!is_ig])
+  is_ig & !has_direct
+}
+
+#' Build the notes text for a direct-suppressed row: names the recipe that
+#' recovers the missing Direct component when one of the (already
+#' Direct-leg-scoped, see C1(a)) suggestions covers this row's year, or a
+#' generic fallback when no such recipe was found.
+#' @noRd
+.direct_suppressed_note <- function(year, suggestions) {
+  matching <- Filter(function(s) {
+    ay <- s$available_years
+    !is.null(ay) && length(ay) == 2L && year >= ay[1] && year <= ay[2]
+  }, suggestions)
+  if (length(matching) == 0L) {
+    return(paste(
+      "Direct component is unavailable through this basis for this year",
+      "(legacy aggregate-only family); no covering recipe found in this",
+      "corpus -- see cog_recipes()."
+    ))
+  }
+  ids <- sort(unique(vapply(matching, function(s) s$recipe_id, character(1))))
+  sprintf(
+    "Direct component is unavailable through this basis for this year; recover it via recipe = '%s' (see cog_recipes()).",
+    paste(ids, collapse = "', '")
+  )
+}
+
+#' @noRd
+.notes_column <- function(result, direct_suppressed = NULL, suggestions = list()) {
   n <- nrow(result)
   if (n == 0L) return(character(0))
-  parts <- vector("list", 2L)
+  parts <- vector("list", 3L)
   agg <- result[["aggregate_fallback"]]
   parts[[1]] <- if (!is.null(agg)) {
     ifelse(agg %in% TRUE,
@@ -430,6 +562,14 @@ cog_spending <- function(govid, years, category = NULL,
   } else {
     rep(NA_character_, n)
   }
+  parts[[3]] <- if (!is.null(direct_suppressed) && any(direct_suppressed)) {
+    vapply(seq_len(n), function(i) {
+      if (!isTRUE(direct_suppressed[i])) return(NA_character_)
+      .direct_suppressed_note(result$year[i], suggestions)
+    }, character(1))
+  } else {
+    rep(NA_character_, n)
+  }
   out <- character(n)
   for (i in seq_len(n)) {
     pieces <- vapply(parts, `[[`, character(1), i)

inst/schemas/provenance-v1.json

@@ -21,6 +21,10 @@
       "type": ["string", "null"],
       "description": "How the intergovernmental leg was assembled; null for 'direct'."
     },
+    "expenditure_concept_direct_suppressed": {
+      "type": "boolean",
+      "description": "TRUE when expenditure_concept = 'total' and at least one requested (year, category) has intergovernmental rows but NO Direct rows in this corpus (typically a legacy aggregate-only family) -- those result rows report the intergovernmental leg alone, not Direct + IG. Always FALSE for expenditure_concept = 'direct'. See the affected rows' `notes` for the recovering recipe, if any."
+    },
     "harmonization":  { "type": "object" },
     "recipe":         { "type": ["object", "null"] },
     "suggestions":    { "type": "array" },

man/cog_spending.Rd

@@ -63,13 +63,28 @@ from prior releases. `"total"` additionally UNIONs in the
 intergovernmental leg -- payments to local governments (`M` codes) and
 to the state government (`L` codes, excluding the `L--` family-total
 rollup) -- so results gain rows with `spend_subtype ==
-"intergovernmental"`. Mutually exclusive with `recipe` (a recipe
+"intergovernmental"`. Requires the active corpus's `summary_categories`
+to carry M/L rows (added by cog_pipeline PR #59); aborts with class
+`uscogdata_ig_categories_unsupported` on an older corpus rather than
+silently under-reporting. Mutually exclusive with `recipe` (a recipe
 already defines its own component codes). **Do not sum `"total"`
 results across levels of government** (e.g. state + county + city):
 a state's `M12` payment to a school district is the same dollar the
 district reports as its own direct `E12`, so summing both double-counts
 it. This matters in particular with [cog_geographic_rollup()], which
-sums across exactly that kind of multi-layer government set.}
+sums across exactly that kind of multi-layer government set.
+
+In the legacy wide era (<= FY2011), some functions are published ONLY
+as an aggregate-flagged family total (e.g. Corrections' `E04`/`E05`
+split), which the Direct leg excludes by construction but the IG leg
+deliberately keeps (see `inst/sql/24-ig_long.sql`). For a `"total"`
+query, any (year, category) where this leaves intergovernmental rows
+with NO Direct counterpart is flagged: the affected rows' `notes`
+name the harmonization recipe that recovers the missing Direct
+component (when one exists), and
+`provenance$expenditure_concept_direct_suppressed` is `TRUE` -- the
+figure in those rows is the intergovernmental leg alone, not Direct +
+IG.}
 }
 \value{
 Tibble with columns `year`, `canonical_govid`, `gov_name`,

tests/testthat/helper-fixture.R

@@ -57,3 +57,37 @@ with_doctored_schema_version <- function(version, code) {
   }, add = TRUE)
   force(code)
 }
+
+# Copy the bundled fixture to a temp dir with summary_categories.parquet
+# rewritten to drop every M/L (intergovernmental) row, then run `code`
+# against it with a clean session (mirrors with_fixture_corpus()/
+# with_doctored_schema_version()). Models a real pre-cog_pipeline-PR#59
+# corpus: the 66 M/L category rows shipped with NO schema_version bump (see
+# C2 in the expenditure-concept review), so schema_version is left
+# untouched here -- only the category data itself is rolled back.
+with_corpus_missing_ig_categories <- function(code) {
+  src <- fixture_corpus_path()
+  tmp <- withr::local_tempdir(.local_envir = parent.frame())
+  file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
+
+  cats_path <- file.path(tmp, "data", "summary_categories.parquet")
+  filtered_path <- file.path(tmp, "data", "summary_categories_filtered.parquet")
+  write_con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(write_con, shutdown = TRUE), add = TRUE)
+  DBI::dbExecute(write_con, sprintf(
+    "COPY (SELECT * FROM read_parquet(%s) WHERE LEFT(item_code, 1) NOT IN ('M', 'L'))
+     TO %s (FORMAT PARQUET)",
+    uscogdata:::.sql_lit_chr(cats_path), uscogdata:::.sql_lit_chr(filtered_path)
+  ))
+  file.remove(cats_path)
+  file.rename(filtered_path, cats_path)
+
+  old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
+  uscogdata:::cog_close()
+  Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
+  on.exit({
+    uscogdata:::cog_close()
+    if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
+  }, add = TRUE)
+  force(code)
+}

tests/testthat/test-views.R

@@ -301,6 +301,23 @@ test_that(".harmonization_view_files guard is necessary: registration against a
   on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
   DBI::dbExecute(con2, .read_view_sql("10-long.sql"))
   expect_error(DBI::dbExecute(con2, .read_view_sql("25-ig_long_harmonized.sql")))
+
+  # Reconciling this test with the C2 guard (expenditure-concept review):
+  # `ig_annotated`/`spending_annotated` registering cleanly above proves
+  # only that CREATE VIEW binds against a `summary_categories` with no M/L
+  # rows at all (this synthetic corpus's own summary_categories has a
+  # single E36 row, see the COPY above) -- a LEFT JOIN never fails to
+  # resolve regardless of what the joined-to table contains. It does NOT
+  # mean querying expenditure_concept = "total" against this shape is safe:
+  # exactly this corpus (schema_version reported as supported, but
+  # summary_categories predates the M/L rows cog_pipeline PR #59 added) is
+  # what .require_ig_categories() exists to catch at the *verb* level,
+  # since PR #59 shipped those rows with no schema_version bump. Confirm
+  # the new runtime guard actually fires against this same `con`.
+  expect_error(
+    uscogdata:::.require_ig_categories(con),
+    class = "uscogdata_ig_categories_unsupported"
+  )
 })
 
 test_that("spending_long filters to E/F/G/K prefixes and excludes aggregates", {