refactor: split .suppressed_components() into R/suppression.R (#9)

R/suggestions.R crossed the project's 400-line limit (424 lines).
Pure move of .suppressed_components() and its roxygen block per the
plan's Task 5 Step 2 remedy; no logic, SQL, or wording changed.

.Rbuildignore

@@ -10,3 +10,10 @@
 ^\.gitignore$
 \.gitkeep$
 ^vignettes$
+^specs$
+^plans$
+^doc$
+^Meta$
+^\.gitea$
+^CLAUDE\.md$
+^\.superpowers$

.gitea/workflows/ci.yml

@@ -11,6 +11,21 @@ jobs:
     steps:
       - name: Install system libraries and Node.js (required by actions/checkout)
         run: |
+          # Switch apt to HTTPS mirrors. Measured from this runner on
+          # 2026-08-04: the SAME index file takes 20.1s over http:// and 3.1s
+          # over https://. apt fetches many indexes serially, so http:// does
+          # not read as "slow" -- it reads as a hang (zero bytes in
+          # /var/cache/apt/archives after 3+ minutes, apt's http workers parked
+          # in S state). rocker/r-ver:4.4 already ships ca-certificates and
+          # apt 2.8.3 has the https method built in, so nothing needs to be
+          # installed over http first to bootstrap this.
+          # `|| true` because the step runs under `sh -e`: on an image whose
+          # sources live in the other location, the missing-file sed must not
+          # kill the job.
+          sed -i -E 's#http://(archive|security)\.ubuntu\.com#https://\1.ubuntu.com#g' \
+            /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true
+          sed -i -E 's#http://(archive|security)\.ubuntu\.com#https://\1.ubuntu.com#g' \
+            /etc/apt/sources.list 2>/dev/null || true
           apt-get update -qq
           apt-get install -y --no-install-recommends \
             nodejs git \

.gitignore

@@ -9,3 +9,6 @@ docs/
 /Meta/
 .DS_Store
 /.quarto/
+
+# SDD working artifacts (ledger, briefs, review packages) — plans/ stays tracked
+.superpowers/sdd/

CLAUDE.md

@@ -28,8 +28,23 @@ USCOGDATA_URL (local path or https://)
 - `R/session.R` — `cog_open()`, `cog_close()`, `.ensure_session()`, `.coerce_govid_input()`
 - `R/manifest.R` — `.fetch_or_cache_manifest()`, `.is_local_path()` (local paths bypass HTTP/cache)
 - `R/views.R` — `.register_views()` (substitutes `{url}` into SQL files at `inst/sql/`)
-- `inst/sql/` — 7 SQL view definitions: `long`, `spending_long`, `revenue_long`, `canonical_fips_xwalk`, `summary_categories`, `spending_annotated`, `revenue_annotated`
+- `inst/sql/` — **23** SQL view definitions (measured), numbered by load order
+  (`10-` through `46-`): the `*_long` layer (`long`, `spending_long`,
+  `revenue_long`, `ig_long`, `balance_long`, plus `_harmonized` variants of
+  `spending_long`/`revenue_long`/`ig_long`), the `*_annotated` layer
+  (`spending_annotated`, `revenue_annotated`, `ig_annotated`,
+  `balance_annotated`, plus `_harmonized` variants of `spending_annotated`/
+  `revenue_annotated`/`ig_annotated`), and metadata views
+  (`canonical_fips_xwalk`, `summary_categories`, `gov_population_yearly`,
+  `harmonization_map`, `harmonization_recipes`, `series_breaks_pq`,
+  `representation`, `code_set`)
 - `R/spending.R` / `R/revenue.R` — `cog_spending()` / `cog_revenue()` via shared `.verb_spendrev()`
+- `R/balances.R` — `cog_balances()`. A third money-adjacent verb, but returns a
+  **stock** (a balance at a point in time) rather than a **flow** (activity
+  over a fiscal year), so it does NOT route through `.verb_spendrev()` and has
+  no `expenditure_concept`/`revenue_concept`/`complete`/`subtype` arguments.
+  `R/balance_caveats.R` attaches `provenance$balance_caveats` (GAAP-vs-gross
+  disclosure + measured per-subtype coverage windows).
 - `R/rollup.R` — `cog_geographic_rollup()` (accepts named list of govids by layer)
 - `R/peers.R` — `cog_find_peers()` + `cog_peer_compare()`
 - `R/search.R` — `cog_gov_search()` (name pattern, state, type filters)
@@ -45,28 +60,31 @@ USCOGDATA_URL (local path or https://)
 Any value without `://` is treated as a local path by `.is_local_path()` and reads
 `manifest.json` directly from disk (no HTTP, no TTL cache).
 
-## Current State (2026-04-27)
+## Current State (2026-08-03)
 
 **Version:** 0.1.0 (pre-release)  
-**Branch:** `main`, commit `d65e9fe`  
-**Tests:** 181 PASS / 0 FAIL / 0 SKIP  
+**Branch:** `feat/cog-balances-25`, commit `fde62eb`  
+**Tests:** 788 PASS / 0 FAIL / 0 SKIP / 0 WARN (measured `testthat::test_local()`, 2026-08-03, after the final-review fix wave)  
 **CI:** Gitea Actions green (`.gitea/workflows/ci.yml`)
 
 ### Completed (Tasks 2.1–2.7)
 
-All 8 exported verbs implemented and tested:
-`cog_spending`, `cog_revenue`, `cog_explain`, `cog_geographic_rollup`,
-`cog_find_peers`, `cog_peer_compare`, `cog_gov_search`, `cog_mirror`,
-plus `cog_categories`.
+All **14** exports implemented and tested (measured from `NAMESPACE`):
+`cog_spending`, `cog_revenue`, `cog_balances`, `cog_explain`,
+`cog_geographic_rollup`, `cog_find_peers`, `cog_peer_compare`,
+`cog_gov_search`, `cog_mirror`, `cog_categories`, `cog_recipes`,
+`cog_manifest`, `cog_basket_resolution`, `cog_basket_unresolved`.
 
-Bundled fixture corpus at `inst/extdata/fixture_corpus/` (3.6 MB, years
-2019+2020, all 50 states). Tests run fully offline — no credentials needed.
+Bundled fixture corpus at `inst/extdata/fixture_corpus/` (years
+2011, 2012, 2019, 2020 — measured via DuckDB `read_parquet(hive_partitioning=1)`,
+2026-08-03; all 50 states). Tests run fully offline — no credentials needed.
 
 ### Remaining to v0.1 release
 
-1. **Task 2.8 — Docs:** roxygen `@param`/`@return`/`@examples` on all exports;
-   full `README.md`; `_pkgdown.yml`; `devtools::document()` + `pkgdown::build_site()`.
-   Vignettes can be stubbed for v0.1.
+1. **Task 2.8 — Docs:** mostly done — all 14 exports have a `man/*.Rd`,
+   `README.md` and `_pkgdown.yml` exist, and `vignettes/` carries
+   `total-spending.Rmd` + `population-denominators.Rmd`. Outstanding:
+   `pkgdown::build_site()` has never been run (no `docs/`).
 
 2. **Phase 3 — cog_explorer bridge:** create
    `cog_explorer/examples/hello_world_uscogdata.Rmd` (installs from Gitea, runs
@@ -99,6 +117,10 @@ devtools::test()
 - All verbs call `.ensure_session()` first, then query via `DBI::dbGetQuery()`
 - Return value is always a `tbl_df` with a `provenance` attribute
 - govid inputs always go through `.coerce_govid_input()` (accepts character or data frame)
-- SQL lives in `inst/sql/` — never inline SQL strings in R files
+- SQL has two layers. **View definitions** live in `inst/sql/` and are
+  registered by `.register_views()`, which globs the directory in sorted order
+  and substitutes `{url}`. **Query construction** is inline `sprintf()` in R
+  (`.build_verb_sql()`, `.run_recipe()`, `.attach_per_capita()`). Add a view as
+  a numbered `.sql` file; build a query in R.
 - No arrow dependency — DuckDB reads parquet natively
 - `withr` is a Suggests-only dep; only used in tests

DESCRIPTION

@@ -31,5 +31,5 @@ Suggests:
 Config/testthat/edition: 3
 VignetteBuilder: knitr
 RoxygenNote: 7.3.3
-MinCorpusSchema: 3
-MaxCorpusSchema: 3
+MinCorpusSchema: 4
+MaxCorpusSchema: 5

NAMESPACE

@@ -1,5 +1,6 @@
 # Generated by roxygen2: do not edit by hand
 
+export(cog_balances)
 export(cog_basket_resolution)
 export(cog_basket_unresolved)
 export(cog_categories)
@@ -7,7 +8,9 @@ export(cog_explain)
 export(cog_find_peers)
 export(cog_geographic_rollup)
 export(cog_gov_search)
+export(cog_manifest)
 export(cog_mirror)
 export(cog_peer_compare)
+export(cog_recipes)
 export(cog_revenue)
 export(cog_spending)

NEWS.md

@@ -1,5 +1,242 @@
 # uscogdata 0.1.0 (development)
 
+## Signposting now catches partially-suppressed categories
+
+* A coverage suggestion used to fire only when a category returned **no rows
+  at all** in a requested year. That missed the more dangerous case: a
+  category that still returns rows while silently dropping component codes
+  the wide era publishes only as aggregates (#9). `cog_spending(category =
+  "Public Welfare")` for FY2011 returned a plausible figure that omitted
+  `E67`/`E68` entirely -- for Los Angeles County, $2,075,461,000 of a true
+  $5,261,404,000, a 39% understatement, with `provenance$suggestions` empty.
+* Suggestions now also fire on **partial** coverage, and every suggestion
+  carries `trigger` (`"empty_year"` or `"suppressed_component"`),
+  `suppressed_amount`, `suppressed_years` and `suppressed_codes`, so a caller
+  can see how much is missing and decide whether to re-run with the recipe.
+* `cog_revenue()` gets the same fix through the shared verb path. Alaska's
+  FY2011 `Miscellaneous Revenue` reported $943,842,000 while dropping
+  $1,899,995,000 of aggregate-published `U4-` rents and royalties.
+* The trigger stays recipe-driven, so it only fires where a harmonization
+  recipe actually exists to name the fix. `higher_ed_e18_wide` and
+  `general_gov_e89_wide` stay silent in every year measured on the bundled
+  fixture, because their components are ordinary classified leaves even
+  pre-2012.
+* The `suppressed_component` trigger (and any `suppressed_amount`/
+  `suppressed_codes` an `empty_year` fire also carries) is scoped to the
+  calling verb's own flow family: `cog_spending()` only ever measures E/F/G
+  component dollars, `cog_revenue()` only T/A/U/B/C/D. A component from the
+  OTHER flow family reports `suppressed_amount = 0` rather than a fabricated
+  claim. The `empty_year` trigger itself is not flow-scoped -- a category
+  belonging to the other flow (e.g. `cog_spending(category = "IG Local")`)
+  still returns zero rows and can still fire, in any year including modern
+  ones, naming the recipe whose own generic join finds real data for this
+  government. That is a mis-scoped query, not a corpus-format gap, so its
+  `suppressed_amount` is correctly 0.
+
+## New: `cog_balances()` for cash-and-security holdings
+
+* New `cog_balances()` exposes the 14 cash-and-security holding codes
+  (`category_type = "balance"`): fund balances, retirement system holdings and
+  insurance trust balances (#25). Holdings are a stock, not a flow, so the verb
+  has no `expenditure_concept` / `revenue_concept` / `complete` arguments, and
+  no `subtype` argument either -- for holdings, `category` is a strict
+  coarsening of `balance_subtype`, so `category = "Fund Balances"` is exactly
+  the `general` family (`W01`/`W31`/`W61`).
+* `cog_balances()` results carry `provenance$balance_caveats`, recording that
+  Census holdings are gross rather than GAAP fund balance, and the measured
+  coverage window of each subtype family.
+
+## Multi-government aggregates now disclose their reporting coverage
+
+* The Census of Governments is a **complete census only in years ending in 2
+  and 7**; every other year is a sample, and the sample varies enormously. On
+  the bundled fixture, Wisconsin's 608-city universe rolls up **597**
+  governments in FY2012 and **112** in FY2019 — an 18%-to-98% swing the
+  return value said nothing about, so a statewide total resting on a fifth of
+  the universe looked exactly like one resting on all of it.
+* `cog_geographic_rollup()`, `cog_peer_compare()` and `cog_find_peers()` gain
+  `coverage`:
+
+  | value | effect |
+  |---|---|
+  | `"all"` (default) | every unit that reported that year — unchanged behaviour |
+  | `"census"` | census years only; aborts if the range holds none rather than returning nothing |
+  | `"consistent"` | only units reporting in *every* requested year — a balanced panel |
+
+* **Regardless of mode**, every result now carries `provenance$coverage` with
+  per-year `n_units_reporting`, `n_units_expected` and `is_census_year`, plus
+  `provenance$coverage_mode`. `cog_explain()` prints a "Reporting coverage"
+  section. So the default mode can no longer mislead silently.
+* `is_census_year` is a statement about the **survey calendar**, never a claim
+  of completeness: FY1967 is a census year in which only 97 of Wisconsin's 608
+  cities report. `n_units_reporting` is the number that tells the truth.
+* On `cog_peer_compare()` the target is exempt from `"consistent"` balancing —
+  it is the subject of the comparison, not a member of the cohort — and the
+  `summary_*` quantiles are computed after the filter, so they describe the
+  cohort actually returned. `n_units_reporting` counts peers only, against the
+  cohort size.
+* On `cog_find_peers()`, `coverage` governs the cohort **vintage** when `year`
+  is `NULL`: `"census"` snaps to the most recent census year with an observed
+  population, so a cohort is not built from a sample year in which most of the
+  candidate universe is absent.
+
+## `complete = TRUE`: absent cells, labelled with why they are absent
+
+* `cog_spending()` and `cog_revenue()` gain `complete`, defaulting to `FALSE`
+  (today's behaviour). With `complete = TRUE` the requested grid is filled
+  from the corpus's `code_set` table and every row carries a new
+  `value_source` column:
+
+  | `value_source` | meaning | `amt_nominal` |
+  |---|---|---|
+  | `reported` | the corpus carries this cell | as published |
+  | `census_zero` | dense-source year (≤ FY2011), cell absent — Census published `$0` | `0` |
+  | `not_reported` | sparse-source year (≥ FY2012), cell absent — unknown | `NA` |
+
+  The `NA` is deliberate and is the whole point: filling a modern absence
+  with `0` would invent data, which is precisely the error the corpus's
+  representation contract exists to prevent.
+* This restores information the reader lost when the corpus was sparsified
+  (`SB194`, cog_pipeline#64) — a wide-era query whose cells were all `$0`
+  had begun returning nothing at all — and improves on what came before it,
+  since the pre-sparsification corpus could not distinguish a published zero
+  from an unreported cell either.
+* The grid is scoped to each government's **own type**, so a county is never
+  filled with cells only a state can report.
+* Needs a corpus published from 2026-07-29 onward (when `representation` and
+  `code_set` began shipping); aborts with class
+  `uscogdata_representation_unavailable` otherwise. Gated on the manifest
+  listing those tables rather than on `schema_version`, which was never
+  bumped for the change. Not available with `recipe` or
+  `expenditure_concept = "total"` — neither draws its cells from `code_set`.
+* `provenance$completion` reports `applied`, `rows_filled`, and the per-year
+  `absence_means` rule; `cog_explain()` prints a "Completion" section.
+
+## Corpus-wide series breaks now reach users (`corpus_break_refs`)
+
+* Four catalogued series breaks carry `fin_code = "ALL"` — caveats about the
+  corpus as a whole rather than about one item code. `series_break_refs` is
+  built by matching `fin_code` against the item codes in the result, and no
+  row's `item_code` is ever the literal `"ALL"`, so **none of them could ever
+  be surfaced**: `SB085` (dollar precision across the 1976/1977 boundary),
+  `SB087` (imputation exclusion from FY2002), `SB194` (the dense → sparse
+  representation change at FY2012) and `SB086` (the government id scheme
+  change at FY2017).
+* Provenance gains `corpus_break_refs`, selected on the break-year window
+  alone and disjoint from `series_break_refs` by construction, so a consumer
+  can tell a whole-result caveat from a break in one series. `cog_explain()`
+  prints them under their own "Corpus-wide caveats" heading. cog-api passes
+  provenance through verbatim, so the field appears there without an API
+  change.
+* `SB194` is the one that made this urgent: a query spanning FY2011 → FY2012
+  crosses the boundary where an absent cell stops meaning "Census published
+  `$0`" and starts meaning "not reported", and until now nothing said so.
+
+## Bundled fixture regenerated against the sparsified corpus
+
+* `inst/extdata/fixture_corpus/` now tracks the corpus published on
+  2026-07-29 (`pipeline_commit 83f9715`, schema v6). The wide era no longer
+  stores explicit zeros: FY2011 fell from 2,864,212 rows to 496,004, of
+  which none are `$0`. **Absence now means two different things** — in a
+  `dense_source` year (≤ FY2011) an absent cell means Census published `$0`;
+  in a `sparse_source` year (≥ FY2012) it means not reported. The corpus
+  carries that rule in two new tables the fixture now ships,
+  `representation.parquet` and `code_set.parquet`, alongside
+  `census_collection_coverage.parquet` and `lineage_events.parquet`
+  (all ten publish-tree metadata tables, up from six). Catalogued upstream
+  as series break `SB194`.
+* `cog_categories()` gains an `assistance` spending subtype: the J-prefix
+  aid/benefit codes (`J19`, `J67`, `J68`, `J85`) are categorised now that
+  the upstream crosswalk covers every flow code carrying dollars.
+* Two consequences worth knowing about, both visible in provenance rather
+  than in returned dollars. The harmonization block's `na_rows_excluded`
+  counts only rows that exist, so wide-era codes that were zero-padded no
+  longer appear there. Coverage-gap `suggestions` are presence-based for the
+  same reason, so a recipe whose component codes were all `$0` for a given
+  government-year is no longer suggested for it.
+* `tests/testthat/test-fixture-vintage.R` pins these structural facts, so a
+  fixture left behind by a future publish fails loudly instead of letting the
+  suite pass against a corpus that no longer exists.
+
+## Breaking: corpus schema_version 4 (Phase P canonical ids)
+
+* The package now requires corpus `schema_version = 4` (`MinCorpusSchema` /
+  `MaxCorpusSchema` in `DESCRIPTION` are both `4`); older corpora built
+  against schema 3 are rejected by `cog_open()` with a clear version-mismatch
+  error. `canonical_govid` is now uniformly 12 characters across every
+  vintage the corpus covers (previously a mix of 9-char legacy ids and
+  12-char FIPS ids depending on source year) — **every hardcoded
+  `canonical_govid` literal from a pre-Phase-P corpus is now invalid** and
+  must be re-resolved via `cog_gov_search()` or the new `canonical_alias`
+  lookup table. `canonical_fips_xwalk` gains four columns
+  (`legacy_govs_id`, `census_geoid`, `id_source`; `confidence` is renamed to
+  `pop_confidence`) and a companion `canonical_alias` table ships in the
+  corpus for mapping legacy/alternate ids onto the current canonical
+  namespace. The bundled fixture corpus (`inst/extdata/fixture_corpus/`) has
+  been regenerated against the Phase P publish tree, now ships the full
+  `canonical_fips_xwalk` and `canonical_alias` master tables alongside the
+  2019-2020 long partitions, and is reproducible via
+  `data-raw/regenerate_fixture_corpus.R`.
+
+## Clearer errors when `USCOGDATA_URL` is unconfigured or returns non-JSON
+
+* `cog_open()` now aborts with the `uscogdata_url_not_configured` error
+  class when the resolved corpus URL still contains the placeholder
+  `REPLACE_WITH_SHARE_TOKEN` sentinel (or is empty). The message lists both
+  remediation paths (`Sys.setenv(USCOGDATA_URL = ...)` and
+  `options(uscogdata.url = ...)`) and points at the bundled fixture for
+  offline testing. Previously the package proceeded to fetch the placeholder
+  URL, cached the resulting HTML welcome page, and failed downstream with a
+  cryptic `jsonlite` lexical-error.
+* `.fetch_or_cache_manifest()` now parses the HTTP response body before
+  persisting it. Non-JSON responses (login pages, 404 HTML) raise
+  `uscogdata_invalid_manifest` with the URL, Content-Type, and underlying
+  parse error — and never write to the on-disk cache.
+* Manifest cache writes are now atomic (write to `manifest.json.tmp.<pid>`
+  in `cache_dir`, then `file.rename` over the target), so an interrupted
+  fetch cannot replace a previously-good cache.
+* Existing caches with non-JSON content (poisoned by the prior code path)
+  are silently refetched instead of returning a parse error to the caller.
+* Local `USCOGDATA_URL` paths whose `manifest.json` is not valid JSON now
+  surface the same `uscogdata_invalid_manifest` class with file context.
+
+## Per-capita denominators now use per-year Census F-33 population
+
+* `cog_spending()` and `cog_revenue()` previously divided all years' amounts
+  by a single ACS 2018-2022 estimate (`canonical_fips_xwalk.population_acs`),
+  producing biased per-capita values for time-series analysis. They now
+  divide by the F-33 `population` recorded on each gov-year via the new
+  `gov_population_yearly` view. Result tibbles gain a `pop_source` column
+  with values `"census_f33"` or `"unavailable"`. `notes` is updated to
+  concatenate multiple notes with `"; "`.
+
+## Peer cohorts can be set to a chosen year
+
+* `cog_find_peers()` adds a `year` argument (default: most recent year for
+  which the target has an observed population in `gov_population_yearly`).
+  The returned column previously named `population_acs` is now `population`
+  and reflects the cohort year's vintage. The cohort year is attached to the
+  returned tibble as `attr(x, "cohort_year")`.
+* `cog_peer_compare()` now stamps a `cohort_year` column on its result (read
+  from the peers tibble's attribute) and records `cohort_year` plus
+  `cohort_govids` in provenance. When the caller supplies a bare character
+  vector instead of a `cog_find_peers()` result, `cohort_year` is `NA`.
+
+## Rollups exclude govs missing population
+
+* `cog_geographic_rollup(per_capita = TRUE)` drops rows whose government has
+  `pop_source == "unavailable"` and records the dropped govids in
+  `provenance$rollup$excluded_govids`. This excludes special districts
+  (type 4) and school districts (type 5) from per-capita rollups by design.
+
+## New: vignette and provenance metadata
+
+* New vignette `population-denominators` covers the four population sources,
+  the type-4/5 coverage gap, the popyear quirk, and how to build moving-window
+  peer cohorts manually.
+* Provenance gains `transformations$per_capita$popyear_range` and
+  `pop_source_counts`. `cog_explain()` renders both.
+
 ## New features
 
 * `cog_gov_search()` gains a **basket mode**: passing vector `name`

R/balance_caveats.R

@@ -0,0 +1,122 @@
+# R/balance_caveats.R
+#
+# The four caveats from cog_pipeline/docs/data_dictionary.md § Cash and
+# security holdings. Each one silently invalidates an obvious analysis, so
+# they travel in provenance (machine-readable, for cog-api#26) rather than
+# living only in prose.
+#
+# Two of the four are already carried by the code-driven series-break
+# builders and are deliberately NOT duplicated here:
+#   * SB195/SB196 -- X40/X41 book -> market at FY2002 -- fire via
+#     series_break_refs on the recipe path, the only path that observes those
+#     codes.
+# What remains is the GAAP distinction (a constant) and the coverage windows
+# (measured, never hardcoded, so they stay correct as the corpus grows).
+
+#' Per-subtype observed year extents, plus which requested families are
+#' truncated relative to the requested span.
+#' @noRd
+.balance_caveats <- function(con, codes_observed, years) {
+  cw <- .balance_coverage_windows(con)
+
+  observed_subtypes <- if (length(codes_observed) == 0L) {
+    character(0)
+  } else {
+    DBI::dbGetQuery(con, sprintf(
+      "SELECT DISTINCT balance_subtype FROM summary_categories
+       WHERE item_code IN (%s) AND balance_subtype IS NOT NULL",
+      .sql_lit_chr(codes_observed)
+    ))$balance_subtype
+  }
+
+  # A family is "truncated" when the caller asked for years outside the span
+  # that family actually covers -- the FY2016 employee-retirement termination
+  # and the FY2021 end of the W family are both this shape.
+  truncated <- character(0)
+  if (length(years) > 0L) {
+    for (s in observed_subtypes) {
+      w <- cw[[s]]
+      if (is.null(w)) next
+      if (max(years) > w[2] || min(years) < w[1]) truncated <- c(truncated, s)
+    }
+  }
+
+  list(
+    not_gaap = TRUE,
+    not_gaap_note = paste0(
+      "Census holdings are gross -- no liabilities are netted -- and are NOT ",
+      "GAAP fund balance. A reserve ratio built from them overstates what is ",
+      "actually available."
+    ),
+    coverage_window = cw,
+    truncated = sort(unique(truncated))
+  )
+}
+
+#' Per-subtype [min year, max year] extents for EVERY balance subtype in the
+#' mounted corpus, memoised for the session.
+#'
+#' The query carries no govid and no year predicate -- its answer is a property
+#' of the mounted corpus alone and cannot change between calls -- but it scans
+#' the whole of `balance_long`, which measured 35% of `cog_balances()` runtime
+#' on the bundled fixture and would be a per-request throughput ceiling once
+#' cog-api#26 serves this verb over HTTP. Memoised in `.uscogdata_env` and
+#' invalidated by `cog_close()`, the same pattern as `.uscogdata_env$manifest`.
+#'
+#' Scope is deliberately corpus-wide rather than query-scoped: a caller asking
+#' "is there a family I missed?" needs every window. The observed-scoped field
+#' is `truncated`. Documented as such in inst/schemas/provenance-v1.json.
+#' @noRd
+.balance_coverage_windows <- function(con) {
+  cached <- .uscogdata_env$balance_coverage_windows
+  if (!is.null(cached)) return(cached)
+
+  windows <- DBI::dbGetQuery(con,
+    "SELECT c.balance_subtype AS subtype,
+            MIN(l.year) AS year_min,
+            MAX(l.year) AS year_max
+     FROM balance_long l
+     JOIN summary_categories c USING (item_code)
+     WHERE c.balance_subtype IS NOT NULL
+     GROUP BY 1
+     ORDER BY 1"
+  )
+
+  cw <- stats::setNames(
+    lapply(seq_len(nrow(windows)),
+           function(i) as.integer(c(windows$year_min[i], windows$year_max[i]))),
+    windows$subtype
+  )
+  .uscogdata_env$balance_coverage_windows <- cw
+  cw
+}
+
+#' TRUE the first time `key` is seen this session, FALSE thereafter.
+#' Reset by cog_close().
+#' @noRd
+.balance_caveat_once <- function(key) {
+  seen <- .uscogdata_env$balance_caveats_shown
+  if (is.null(seen)) seen <- character(0)
+  if (key %in% seen) return(FALSE)
+  .uscogdata_env$balance_caveats_shown <- c(seen, key)
+  TRUE
+}
+
+#' Emit at most one message per caveat class per session.
+#' @noRd
+.emit_balance_caveats <- function(caveats) {
+  if (.balance_caveat_once("not_gaap")) {
+    cli::cli_inform(c(
+      "!" = "Census holdings are gross and are {.strong not} GAAP fund balance.",
+      "i" = "No liabilities are netted; a reserve ratio built from them overstates available funds."
+    ))
+  }
+  if (length(caveats$truncated) > 0L &&
+      .balance_caveat_once("coverage_window")) {
+    cli::cli_inform(c(
+      "!" = "Requested years extend beyond what {.val {caveats$truncated}} actually covers.",
+      "i" = "See {.code provenance$balance_caveats$coverage_window}."
+    ))
+  }
+  invisible(NULL)
+}

R/balances.R

@@ -0,0 +1,159 @@
+# R/balances.R
+#
+# Cash and security holdings. A third verb rather than an argument on a money
+# verb because holdings are a STOCK -- a balance at a point in time -- while
+# cog_spending()/cog_revenue() return FLOWS over a fiscal year. The money
+# verbs' whole argument vocabulary (expenditure_concept, revenue_concept,
+# complete=) describes flows and is meaningless here, so this deliberately
+# does NOT route through .verb_spendrev().
+
+#' Cash and security holdings for one or more governments
+#'
+#' Returns Census cash-and-security holdings (`category_type = "balance"`):
+#' fund balances, retirement system holdings and insurance trust balances.
+#'
+#' @section Holdings are not GAAP fund balance:
+#' Census holdings are **gross** -- no liabilities are netted -- so a reserve
+#' ratio built from them overstates what is actually available. They are not
+#' comparable to a GAAP fund balance from an ACFR.
+#'
+#' @param govid Canonical govid(s): a character vector, or a data frame with a
+#'   `canonical_govid` column (e.g. from [cog_gov_search()]).
+#' @param years Integer vector of fiscal years.
+#' @param category Optional character vector of categories to keep. One of
+#'   `"Fund Balances"`, `"Insurance Trust Balances"`,
+#'   `"Retirement System Holdings"`. There is deliberately no `subtype`
+#'   argument: for holdings, `category` is a strict coarsening of
+#'   `balance_subtype` (unlike the money verbs, where the two axes cross), so
+#'   every combination would be either redundant or empty.
+#'   `category = "Fund Balances"` is exactly the `general` family
+#'   (`W01`/`W31`/`W61`). `balance_subtype` is returned, so a finer split is
+#'   one `dplyr::filter()` away.
+#' @param per_capita Divide holdings by population. Note this is a **stock per
+#'   resident** (reserves per person), which is *not* comparable to
+#'   [cog_spending()]'s per-capita figures -- those are a flow per person.
+#' @param adjust_to_year Deflate to this year's dollars (CPI-U).
+#' @param basis Accepted for uniformity with the money verbs, but currently a
+#'   **no-op**: `harmonization_map` carries no balance-code rows, so harmonized
+#'   and raw space are identical for holdings. Reported in
+#'   `provenance$basis_note`.
+#' @param recipe Optional harmonization recipe id (see [cog_recipes()]).
+#'   `"cash_securities_z77_wide"` and `"cash_securities_z78_wide"` bridge the
+#'   wide era to the modern one.
+#'
+#' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
+#'   `balance_subtype`, `category`, `amt_nominal`, `codes_included`,
+#'   `aggregate_fallback`, plus optional `amt_per_capita_nominal` and
+#'   `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when
+#'   `adjust_to_year` is set), and optional `amt_per_capita_real` (only when
+#'   **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no
+#'   nominal per-capita column to deflate otherwise). Amounts are full US
+#'   dollars.
+#'
+#'   Carries a `provenance` attribute matching
+#'   `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports
+#'   `not_gaap`, `not_gaap_note`, `coverage_window` (measured year extents for
+#'   every balance subtype in the mounted corpus, not only the observed ones)
+#'   and `truncated` (the observed subtypes whose coverage falls short of the
+#'   requested years). `expenditure_concept`/`revenue_concept` are `NA` --
+#'   holdings are a stock, not a flow, so neither concept vocabulary applies.
+#' @export
+cog_balances <- function(govid, years, category = NULL,
+                         per_capita = FALSE, adjust_to_year = NULL,
+                         basis = c("harmonized", "raw"), recipe = NULL) {
+  call <- match.call()
+  basis <- match.arg(basis, c("harmonized", "raw"))
+  # Coerce FIRST, validate second: .validate_verb_inputs() asserts
+  # is.character(govid), and a data-frame govid (cog_gov_search() output) has
+  # not been unwrapped yet at this point.
+  govid <- .coerce_govid_input(govid)
+  # The money verbs' validator, reused rather than re-implemented (R/spending.R).
+  # It covers the exact superset cog_balances() needs -- including the
+  # recipe/category mutual-exclusivity guard -- so a second local copy would
+  # only be a place for the two to drift apart. This is the same kind of
+  # helper reuse as .build_verb_sql()/.attach_per_capita() below; it does NOT
+  # route the verb through .verb_spendrev(), which stays deliberately unused
+  # here because its flow vocabulary is meaningless for a stock.
+  .validate_verb_inputs(govid, years, category, per_capita, adjust_to_year,
+                        recipe)
+  years <- as.integer(years)
+  if (!is.null(adjust_to_year)) adjust_to_year <- as.integer(adjust_to_year)
+
+  con <- .ensure_session()
+  .require_balance_support(con)
+  scope <- .check_govids_in_scope(govid)
+
+  basis_note <- paste0(
+    "`basis` has no effect on holdings: harmonization_map carries no ",
+    "balance-code rows, so harmonized and raw space are identical here."
+  )
+
+  manifest <- .uscogdata_env$manifest
+  recipe_block <- NULL
+  category_for_prov <- category
+
+  if (!is.null(recipe)) {
+    .require_schema_v5(con, manifest, "recipe =")
+    .validate_recipe_id(con, recipe)
+    comps <- .recipe_components(con, recipe)
+    recipe_label <- comps$label[[1]]
+    result <- .run_recipe(con, recipe, govid, years)
+    sql <- attr(result, "sql_query")
+    result <- .shape_recipe_result(result, "balance_subtype", recipe_label)
+    recipe_block <- list(
+      recipe_id = recipe, label = recipe_label,
+      components = .df_to_row_list(comps)
+    )
+    category_for_prov <- recipe_label
+  } else {
+    sql <- .build_verb_sql("balance_annotated", "balance_subtype",
+                           govid, years, category,
+                           ig_view = NULL, subtype_scope = NULL)
+    result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
+  }
+
+  # Order matters (matches .verb_spendrev()): per-capita first, so
+  # .attach_real_dollars() deflates the nominal per-capita column into
+  # amt_per_capita_real rather than needing amt_per_capita_nominal recomputed.
+  if (isTRUE(per_capita)) result <- .attach_per_capita(result, con, govid)
+  if (!is.null(adjust_to_year)) {
+    result <- .attach_real_dollars(result, adjust_to_year, per_capita)
+  }
+
+  prov <- .build_provenance(
+    verb = "cog_balances", call = call, govid = govid, years = years,
+    category = category_for_prov, per_capita = per_capita,
+    adjust_to_year = adjust_to_year, result = result, sql = sql,
+    subtype_col = "balance_subtype",
+    basis = basis, basis_note = basis_note,
+    # Neither concept vocabulary applies to a stock.
+    expenditure_concept = NA_character_,
+    revenue_concept = NA_character_,
+    recipe = recipe_block
+  )
+  prov$scope$govids_found   <- scope$found
+  prov$scope$govids_missing <- scope$missing
+
+  prov$balance_caveats <- .balance_caveats(
+    con, prov$codes_summed$observed, years
+  )
+  .emit_balance_caveats(prov$balance_caveats)
+
+  attr(result, "provenance") <- prov
+  result
+}
+
+#' Abort unless the mounted corpus classifies balance codes.
+#'
+#' `balance_subtype` arrived with cog_pipeline #76/#77 without a
+#' schema_version bump, so the check is on the column, not the version.
+#' @noRd
+.require_balance_support <- function(con) {
+  if (.corpus_has_balance_subtype(con)) return(invisible(TRUE))
+  cli::cli_abort(
+    c("This corpus does not classify cash and security holdings.",
+      i = "`summary_categories` has no {.field balance_subtype} column.",
+      i = "Republish from cog_pipeline at #76/#77 or later."),
+    class = "uscogdata_no_balance_support"
+  )
+}

R/basis.R

@@ -0,0 +1,87 @@
+# R/basis.R
+# basis= resolution (harmonized/raw, with v4/v5 dual-accept) and the
+# harmonization exclusion-count block attached to provenance.
+
+#' Resolve the requested `basis` against the active corpus's schema_version.
+#'
+#' On a `schema_version >= 5` corpus, the requested basis is used as-is. On
+#' an older (`schema_version == 4`) corpus, which has no harmonization
+#' tables: a caller who left `basis` at its default (`"harmonized"`, so
+#' `explicit` is `FALSE`) silently gets `"raw"` back, with a note recorded
+#' for provenance; a caller who explicitly asked for
+#' `basis = "harmonized"` gets a hard abort instead of a silent downgrade.
+#'
+#' @param basis `"harmonized"` or `"raw"` (already resolved via `match.arg`).
+#' @param explicit `TRUE` if the caller passed `basis` explicitly (as
+#'   opposed to relying on the default `c("harmonized", "raw")`).
+#' @param manifest The active session's parsed manifest list.
+#' @return List with `basis` (the resolved value) and `note` (character or
+#'   `NA_character_`).
+#' @noRd
+.resolve_basis <- function(basis, explicit, manifest) {
+  schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
+
+  if (schema_version >= 5L) {
+    return(list(basis = basis, note = NA_character_))
+  }
+
+  if (identical(basis, "harmonized") && explicit) {
+    cli::cli_abort(c(
+      "basis = \"harmonized\" requires corpus schema_version >= 5.",
+      x = "Active corpus has schema_version {schema_version}.",
+      i = "Use basis = \"raw\" (the default on this corpus), or point USCOGDATA_URL at a schema_version >= 5 corpus."
+    ), class = "uscogdata_basis_unsupported")
+  }
+
+  list(
+    basis = "raw",
+    note = sprintf(
+      "basis resolved to \"raw\": corpus schema_version %d < 5 (harmonization tables unavailable)",
+      schema_version
+    )
+  )
+}
+
+#' Count + sum item-level rows that basis="harmonized" excludes because they
+#' carry no harmonized_code (discontinued / not-yet-ruled codes) within the
+#' calling verb's crosswalk scope (`subtype_col` values in `subtype_scope` --
+#' the same subtype-membership classification the verb SQL uses, never
+#' item-code prefixes), govids, and years. Only meaningful when the resolved
+#' basis is "harmonized"; returns an applied = FALSE stub otherwise (raw
+#' basis never excludes rows this way).
+#'
+#' The intergovernmental leg is deliberately outside this count even for
+#' expenditure_concept = "total": ig_long_harmonized COALESCEs rather than
+#' drops NULL-harmonized rows, so harmonization never excludes an IG row.
+#' @noRd
+.build_harmonization_block <- function(con, govid, years, resolved,
+                                       subtype_col, subtype_scope) {
+  if (!identical(resolved$basis, "harmonized")) {
+    return(list(
+      applied = FALSE,
+      na_rows_excluded = 0L,
+      na_amount_excluded = 0,
+      note = resolved$note
+    ))
+  }
+
+  sql <- sprintf(
+    "SELECT COUNT(*) AS n, COALESCE(SUM(amt), 0) * 1000.0 AS amt
+     FROM long
+     WHERE canonical_govid IN (%s) AND year IN (%s)
+       AND NOT is_aggregate AND harmonized_code IS NULL
+       AND item_code IN (
+         SELECT item_code FROM summary_categories WHERE %s IN (%s)
+       )",
+    .sql_lit_chr(govid), paste(as.integer(years), collapse = ","),
+    subtype_col, .sql_lit_chr(subtype_scope)
+  )
+  na <- DBI::dbGetQuery(con, sql)
+
+  list(
+    applied = TRUE,
+    na_rows_excluded = as.integer(na$n),
+    na_amount_excluded = as.numeric(na$amt),
+    note = resolved$note
+  )
+}

R/categories.R

@@ -5,12 +5,19 @@
 #' Returns the category taxonomy exposed by the corpus's
 #' `summary_categories` view, grouped to one row per
 #' `(category, subtype)` pair. Use this to discover valid `category`
-#' values for [cog_spending()] / [cog_revenue()] /
+#' values for [cog_spending()] / [cog_revenue()] / [cog_balances()] /
 #' [cog_geographic_rollup()] and to audit which Census item codes feed
 #' each category.
 #'
-#' @param type Either `NULL` (default, return both spending and revenue
-#'   rows), `"spending"`, or `"revenue"`.
+#' `subtype` COALESCEs the crosswalk's three subtype columns, so it carries
+#' `spend_subtype` on expenditure rows, `revenue_subtype` on revenue rows and
+#' `balance_subtype` on balance rows. Note that [cog_balances()] itself takes
+#' no `subtype` argument — for holdings, `category` is a strict coarsening of
+#' `balance_subtype` — but the value is surfaced here because it is the
+#' discovery surface downstream consumers build their vocabulary from.
+#'
+#' @param type Either `NULL` (default, every row: expenditure, revenue and
+#'   balance), `"spending"`, `"revenue"`, or `"balance"`.
 #' @param pattern Optional regex matched case-insensitively against the
 #'   `category` column (e.g. `"Police"` or `"Tax"`).
 #' @return Tibble with columns `category`, `category_type`, `subtype`,
@@ -20,8 +27,8 @@
 cog_categories <- function(type = NULL, pattern = NULL) {
   if (!is.null(type)) {
     if (!is.character(type) || length(type) != 1L ||
-        !type %in% c("spending", "revenue")) {
-      cli::cli_abort('`type` must be NULL, "spending", or "revenue".')
+        !type %in% c("spending", "revenue", "balance")) {
+      cli::cli_abort('`type` must be NULL, "spending", "revenue", or "balance".')
     }
   }
   if (!is.null(pattern) &&
@@ -48,7 +55,7 @@ cog_categories <- function(type = NULL, pattern = NULL) {
 
   sql <- paste(
     "SELECT category, category_type,
-            COALESCE(spend_subtype, revenue_subtype) AS subtype,
+            COALESCE(spend_subtype, revenue_subtype, balance_subtype) AS subtype,
             COUNT(DISTINCT item_code) AS n_codes,
             string_agg(DISTINCT item_code, ',' ORDER BY item_code) AS item_codes
      FROM summary_categories",

R/complete.R

@@ -0,0 +1,149 @@
+# R/complete.R
+#
+# `complete = TRUE` on the money verbs. Fills the requested grid so that a
+# cell the corpus does not carry still appears, labelled with WHY it is
+# missing.
+#
+# The corpus stopped storing the wide era's explicit zeros
+# (cog_pipeline#64, series break SB194), which made absence ambiguous:
+#
+#   <= FY2011  dense_source   absent => Census published $0    (census_zero)
+#   >= FY2012  sparse_source  absent => not reported, unknown  (not_reported)
+#
+# Before sparsification a wide-era query whose cells were all $0 came back as
+# explicit $0 rows; afterwards it came back empty, with nothing to say which
+# of the two meanings applied. This restores that -- and improves on it,
+# because the pre-sparsification corpus could not distinguish the two either.
+#
+# `census_zero` fills carry `amt_nominal = 0`; `not_reported` fills carry NA.
+# That difference is the entire point: writing 0 into a modern absence would
+# invent data, which is the error the representation contract exists to stop.
+
+#' @noRd
+.abort_complete_unsupported <- function(reason, alternative) {
+  cli::cli_abort(c(
+    "{.code complete = TRUE} is not supported for this query.",
+    x = reason,
+    i = alternative
+  ), class = "uscogdata_complete_unsupported")
+}
+
+#' @noRd
+.require_representation <- function(con, manifest) {
+  needed <- c("representation.parquet", "code_set.parquet")
+  missing <- needed[!vapply(needed, function(f) .corpus_has_table(manifest, f),
+                            logical(1))]
+  if (length(missing) == 0L) return(invisible(TRUE))
+  cli::cli_abort(c(
+    "This corpus does not publish the representation contract.",
+    x = "Missing: {.file {missing}}.",
+    i = "{.code complete = TRUE} needs those tables to know whether an absent cell means Census published $0 or means the government did not report.",
+    i = "They ship with corpora published from 2026-07-29 onward; re-point {.envvar USCOGDATA_URL} at a current corpus, or omit {.code complete}."
+  ), class = "uscogdata_representation_unavailable")
+}
+
+#' The cells a government-year COULD carry: every code in force for that
+#' government's own type, mapped through `summary_categories`, restricted to
+#' the calling verb's crosswalk subtype scope (the same subtype-membership
+#' classification the verb SQL itself uses -- e.g. the `primary` concept's
+#' operations/capital/assistance) and (when given) its category filter.
+#'
+#' Scoped by `govs_type` deliberately. Filling against the union of all types
+#' would invent cells that the government can never report -- a county row for
+#' "state IG transfer to school districts" -- and those inventions would then
+#' be indistinguishable from real census zeros.
+#'
+#' `NOT cs.is_aggregate` mirrors `spending_long` / `revenue_long`, which drop
+#' aggregate rows. Without it the grid would offer cells the verb structurally
+#' never returns, so every one of them would fill as a phantom $0.
+#' @noRd
+.completion_grid_sql <- function(subtype_col, govid, years, category,
+                                 subtype_scope) {
+  category_pred <- if (is.null(category)) {
+    ""
+  } else {
+    sprintf("AND c.category IN (%s)", .sql_lit_chr(category))
+  }
+  sprintf(
+    "SELECT DISTINCT
+       cs.year,
+       x.canonical_govid,
+       x.gov_name,
+       c.%1$s AS subtype_value,
+       c.category,
+       r.absence_means
+     FROM code_set cs
+     JOIN canonical_fips_xwalk x ON x.govs_type = cs.type
+     JOIN summary_categories c   ON c.item_code = cs.item_code
+     JOIN representation r       ON r.year = cs.year
+     WHERE x.canonical_govid IN (%2$s)
+       AND cs.year IN (%3$s)
+       AND NOT cs.is_aggregate
+       AND c.category IS NOT NULL
+       AND c.%1$s IN (%4$s)
+       %5$s",
+    subtype_col, .sql_lit_chr(govid),
+    paste(as.integer(years), collapse = ","),
+    .sql_lit_chr(subtype_scope), category_pred
+  )
+}
+
+#' Fill `result` out to the full grid, stamping `value_source` on every row.
+#'
+#' Returns the completed tibble with a `.completion` attribute carrying the
+#' provenance block. Reported rows are passed through untouched -- filling
+#' must never alter or drop what the corpus actually published.
+#' @noRd
+.complete_result <- function(result, con, subtype_col, govid, years, category,
+                             subtype_scope) {
+  grid <- tibble::as_tibble(DBI::dbGetQuery(
+    con, .completion_grid_sql(subtype_col, govid, years, category, subtype_scope)
+  ))
+
+  result$value_source <- rep("reported", nrow(result))
+  if (nrow(grid) == 0L) {
+    attr(result, ".completion") <- list(
+      applied = TRUE, rows_filled = 0L, absence_means = list()
+    )
+    return(result)
+  }
+
+  names(grid)[names(grid) == "subtype_value"] <- subtype_col
+  key <- function(d) {
+    paste(d$year, d$canonical_govid, d[[subtype_col]], d$category, sep = "\r")
+  }
+  missing <- grid[!key(grid) %in% key(result), , drop = FALSE]
+
+  if (nrow(missing) > 0L) {
+    filled <- tibble::tibble(
+      year            = as.integer(missing$year),
+      canonical_govid = as.character(missing$canonical_govid),
+      gov_name        = as.character(missing$gov_name),
+      category        = as.character(missing$category),
+      # census_zero is a value Census published; not_reported is unknown and
+      # must stay NA. Collapsing the two to 0 is the defect, not the fill.
+      amt_nominal     = ifelse(missing$absence_means == "census_zero",
+                               0, NA_real_),
+      codes_included  = NA_character_,
+      aggregate_fallback = NA,
+      value_source    = as.character(missing$absence_means)
+    )
+    filled[[subtype_col]] <- as.character(missing[[subtype_col]])
+    if ("notes" %in% names(result)) filled$notes <- NA_character_
+
+    result <- dplyr::bind_rows(result, filled)
+    result <- result[order(result$year, result$canonical_govid,
+                           result[[subtype_col]], result$category), ,
+                     drop = FALSE]
+  }
+
+  rules <- unique(grid[, c("year", "absence_means")])
+  attr(result, ".completion") <- list(
+    applied = TRUE,
+    rows_filled = nrow(missing),
+    absence_means = stats::setNames(
+      as.list(as.character(rules$absence_means)), as.character(rules$year)
+    )
+  )
+  result
+}

R/config.R

@@ -21,7 +21,30 @@
   .uscogdata_defaults[[key]]
 }
 
-.resolve_url <- function() .cfg("url")
+#' Resolve the corpus URL, guaranteeing the trailing slash the package assumes.
+#'
+#' Every consumer builds locations by CONCATENATION -- `paste0(url,
+#' "manifest.json")` in manifest.R, `paste0(url, e$path)` in mirror.R, and the
+#' parquet glob in views.R -- and mirror.R:104 documents the invariant outright
+#' ('url ends in "/"'). Nothing enforced it, so a URL entered without the slash
+#' failed silently and misleadingly:
+#'
+#'   HTTPS -> ".../downloadmanifest.json"; the host answers with an HTML 404
+#'            page, which lands in the JSON parser as the lexical error
+#'            reported in issue #3 -- pointing the user at "login page / wrong
+#'            share" when the real cause was one missing character.
+#'   local -> ".../corpusdata/long/**/*.parquet" and a DuckDB "No files found".
+#'
+#' Normalizing here fixes every consumer at once, rather than each call site
+#' re-deriving the same invariant. An empty setting is passed through
+#' untouched so manifest.R's "not configured" guard still fires instead of the
+#' value degrading into a bare "/" filesystem root.
+#' @noRd
+.resolve_url <- function() {
+  url <- .cfg("url")
+  if (is.null(url) || !nzchar(url) || grepl("/$", url)) return(url)
+  paste0(url, "/")
+}
 
 .resolve_cache_dir <- function() {
   v <- .cfg("cache_dir")

R/coverage.R

@@ -0,0 +1,107 @@
+# R/coverage.R
+#
+# Reporting-coverage disclosure for the multi-government verbs (uscogdata#13,
+# findings F-020 and F-023).
+#
+# The Census of Governments is a COMPLETE CENSUS only in years ending in 2 and
+# 7. Every other year is a sample, and the sample varies enormously: on the
+# bundled fixture, Wisconsin's 608-city universe reports 597 governments in
+# FY2012 and 112 in FY2019. Summing "whatever reported" across those years is
+# what the verbs have always done -- correctly -- but the return value said
+# nothing about it, so a statewide total resting on 18% of the universe looked
+# exactly like one resting on 98%.
+#
+# Owner's settled design: a `coverage` argument selecting WHICH units to
+# include, plus always-on metadata saying how many there were either way. The
+# principle behind it: using these verbs correctly must not require the caller
+# to know the survey calendar.
+
+# Years ending in 2 or 7 are full censuses of every government; all others are
+# samples.
+.CENSUS_YEAR_ENDINGS <- c(2L, 7L)
+
+#' @noRd
+.is_census_year <- function(years) {
+  as.integer(years) %% 10L %in% .CENSUS_YEAR_ENDINGS
+}
+
+#' @noRd
+.validate_coverage <- function(coverage) {
+  tryCatch(
+    match.arg(coverage, c("all", "census", "consistent")),
+    error = function(e) {
+      cli::cli_abort(
+        "`coverage` must be one of {.val all}, {.val census} or {.val consistent}.",
+        class = "uscogdata_invalid_coverage", parent = e
+      )
+    }
+  )
+}
+
+#' Restrict `years` to census years for `coverage = "census"`.
+#'
+#' Aborts rather than returning an empty result when the requested range holds
+#' no census year: silently handing back zero rows for a query the caller
+#' believes they made is the failure mode this whole issue is about.
+#' @noRd
+.apply_census_years <- function(years, coverage, verb) {
+  if (!identical(coverage, "census")) return(as.integer(years))
+  keep <- as.integer(years)[.is_census_year(years)]
+  if (length(keep) == 0L) {
+    cli::cli_abort(c(
+      "{.code coverage = \"census\"} leaves no years to query.",
+      x = "None of the requested years end in 2 or 7: {.val {sort(unique(as.integer(years)))}}.",
+      i = "Census of Governments years ending in 2 or 7 are complete censuses; all others are samples.",
+      i = "Use {.code coverage = \"all\"} (the default) to keep every requested year, or request a census year."
+    ), class = "uscogdata_no_census_years")
+  }
+  sort(keep)
+}
+
+#' Keep only units that report in EVERY requested year (a balanced panel).
+#'
+#' `id_col` is the government identifier; `keep_ids` are rows exempt from the
+#' filter (the peer-comparison target, which is the subject of the comparison
+#' rather than a member of the cohort being balanced).
+#' @noRd
+.filter_consistent <- function(result, years, id_col = "canonical_govid",
+                               keep_ids = character(0)) {
+  years <- unique(as.integer(years))
+  if (nrow(result) == 0L || length(years) <= 1L) return(result)
+  ids <- setdiff(unique(result[[id_col]]), c(NA, keep_ids))
+  present <- vapply(ids, function(g) {
+    all(years %in% unique(as.integer(result$year[result[[id_col]] == g])))
+  }, logical(1))
+  consistent <- c(ids[present], keep_ids)
+  result[result[[id_col]] %in% consistent | is.na(result[[id_col]]), ,
+         drop = FALSE]
+}
+
+#' Per-year coverage metadata, always attached regardless of mode.
+#'
+#' Built from the REQUESTED years rather than the years present in the result,
+#' so a year in which nothing reported still appears -- with
+#' `n_units_reporting = 0`, which is precisely the disclosure a silently
+#' missing year fails to make.
+#'
+#' `n_units_reporting` describes the result the caller actually received, so
+#' under `coverage = "consistent"` it reports the balanced count. `is_census_year`
+#' is a statement about the SURVEY CALENDAR, never a claim of completeness:
+#' FY1967 is a census year in which only 97 of Wisconsin's 608 cities report.
+#' `n_units_reporting` is the number that tells the truth.
+#' @noRd
+.coverage_table <- function(result, years, n_expected,
+                            id_col = "canonical_govid", rows = NULL) {
+  years <- sort(unique(as.integer(years)))
+  src <- if (is.null(rows)) result else rows
+  reporting <- vapply(years, function(y) {
+    ids <- src[[id_col]][as.integer(src$year) == y]
+    length(unique(ids[!is.na(ids)]))
+  }, integer(1))
+  tibble::tibble(
+    year              = years,
+    n_units_reporting = as.integer(reporting),
+    n_units_expected  = rep(as.integer(n_expected), length(years)),
+    is_census_year    = .is_census_year(years)
+  )
+}

R/explain.R

@@ -51,6 +51,43 @@ cog_explain <- function(result, format = c("print", "list")) {
     cli::cli_text("Category: (all)")
   }
 
+  if (!is.null(prov$basis)) {
+    note <- if (!is.null(prov$basis_note) && !is.na(prov$basis_note)) {
+      sprintf(" (%s)", prov$basis_note)
+    } else {
+      ""
+    }
+    cli::cli_text("Basis: {prov$basis}{note}")
+  }
+
+  # Each verb reports its OWN concept. Both fields are always present (each
+  # defaults to its concept's default), so printing `expenditure_concept`
+  # unconditionally would tell a cog_revenue() caller "Concept: primary",
+  # which names a spending concept their result has nothing to do with.
+  if (identical(prov$verb, "cog_revenue")) {
+    if (!is.null(prov$revenue_concept)) {
+      cli::cli_text("Concept: {prov$revenue_concept} revenue")
+    }
+  } else if (identical(prov$verb, "cog_balances")) {
+    # Both concept fields are deliberately NA here (a stock has no flow
+    # concept). Printing the raw NA reads as a missing value rather than an
+    # intentional one, so say what it means instead.
+    cli::cli_text("Concept: not applicable (holdings are a stock, not a flow)")
+  } else if (!is.null(prov$expenditure_concept)) {
+    concept_note <- if (!is.null(prov$expenditure_concept_note) &&
+                         !is.na(prov$expenditure_concept_note)) {
+      sprintf(" (%s)", prov$expenditure_concept_note)
+    } else {
+      ""
+    }
+    cli::cli_text("Concept: {prov$expenditure_concept}{concept_note}")
+    if (isTRUE(prov$expenditure_concept_direct_suppressed)) {
+      cli::cli_alert_warning(
+        "Direct leg unavailable for at least one requested (year, category) -- affected rows report intergovernmental dollars alone, not Direct + IG. See each row's notes."
+      )
+    }
+  }
+
   cli::cli_h2("Codes observed")
   codes <- prov$codes_summed$observed
   if (length(codes) == 0L) {
@@ -66,6 +103,114 @@ cog_explain <- function(result, format = c("print", "list")) {
     )
   }
 
+  h <- prov$harmonization
+  if (!is.null(h) && isTRUE(h$applied)) {
+    cli::cli_h2("Harmonization")
+    cli::cli_text(
+      "Excluded {h$na_rows_excluded} row(s) with no harmonized_code (${format(h$na_amount_excluded, big.mark = ',')})"
+    )
+  }
+
+  rc <- prov$recipe
+  if (!is.null(rc)) {
+    cli::cli_h2("Recipe")
+    cli::cli_text("{rc$recipe_id}: {rc$label}")
+    comp_lines <- vapply(rc$components, function(x) {
+      sprintf("%s (%s, %s-%s, weight=%s)", x$component_code, x$gov_type_scope,
+              x$year_min, x$year_max, x$weight)
+    }, character(1))
+    cli::cli_ul(comp_lines)
+  }
+
+  if (length(prov$suggestions) > 0L) {
+    cli::cli_h2("Suggestions")
+    sugg_lines <- vapply(prov$suggestions, function(s) {
+      line <- sprintf("%s -- %s (years %s-%s): %s", s$recipe_id, s$label,
+              s$available_years[1], s$available_years[2], s$hint)
+      if (isTRUE(s$suppressed_amount > 0)) {
+        line <- paste0(line, sprintf(" [$%s excluded from %s: %s]",
+          formatC(s$suppressed_amount, format = "f", digits = 0, big.mark = ","),
+          paste0("FY", s$suppressed_years, collapse = ", "),
+          paste(s$suppressed_codes, collapse = ", ")))
+      }
+      line
+    }, character(1))
+    cli::cli_ul(sugg_lines)
+  }
+
+  if (!is.null(prov$coverage) && nrow(prov$coverage) > 0L) {
+    cli::cli_h2("Reporting coverage")
+    cli::cli_text("Mode: {prov$coverage_mode %||% 'all'}")
+    cov <- prov$coverage
+    cli::cli_ul(sprintf(
+      "%d: %d of %d units reporting (%.0f%%) -- %s year",
+      cov$year, cov$n_units_reporting, cov$n_units_expected,
+      100 * cov$n_units_reporting / pmax(cov$n_units_expected, 1L),
+      ifelse(cov$is_census_year, "census", "sample")
+    ))
+    if (any(!cov$is_census_year)) {
+      cli::cli_text(
+        "Note: the Census of Governments is a complete census only in years ending in 2 or 7; every other year is a sample."
+      )
+    }
+  }
+
+  if (isTRUE(prov$completion$applied)) {
+    cli::cli_h2("Completion")
+    cli::cli_text(
+      "Filled {prov$completion$rows_filled} absent cell(s) from the corpus code set."
+    )
+    rules <- prov$completion$absence_means
+    if (length(rules) > 0L) {
+      cli::cli_ul(vapply(names(rules), function(y) {
+        sprintf("%s: an absent cell means %s", y,
+                if (identical(rules[[y]], "census_zero")) {
+                  "Census published $0 (filled as 0)"
+                } else {
+                  "the government did not report (filled as NA, not 0)"
+                })
+      }, character(1)))
+    }
+  }
+
+  if (length(prov$series_break_refs) > 0L) {
+    cli::cli_h2("Series breaks")
+    cli::cli_ul(.series_break_story_lines(prov$series_break_refs))
+  }
+
+  # Kept in a section of its own: these qualify the whole result, so folding
+  # them in with the per-code breaks above would invite reading them as a
+  # caveat about one series.
+  if (length(prov$corpus_break_refs) > 0L) {
+    cli::cli_h2("Corpus-wide caveats")
+    cli::cli_ul(.series_break_story_lines(prov$corpus_break_refs))
+  }
+
+  # Balance results only (NULL on money-verb provenance, so they are
+  # unaffected). This is the ONLY on-demand surface for the GAAP disclosure:
+  # .emit_balance_caveats() fires at most once per session, and is routinely
+  # consumed by a suppressMessages() call or by a knitted chunk nobody reads,
+  # so a caller who deliberately audits a result with cog_explain() must still
+  # be told.
+  bc <- prov$balance_caveats
+  if (!is.null(bc)) {
+    cli::cli_h2("Holdings caveats")
+    if (!is.null(bc$not_gaap_note)) cli::cli_alert_warning(bc$not_gaap_note)
+    if (length(bc$truncated) > 0L) {
+      cli::cli_text(
+        "Requested years extend beyond what these families actually cover:"
+      )
+      cli::cli_ul(vapply(bc$truncated, function(s) {
+        w <- bc$coverage_window[[s]]
+        if (length(w) == 2L) {
+          sprintf("%s: covered %s-%s in this corpus", s, w[1], w[2])
+        } else {
+          s
+        }
+      }, character(1)))
+    }
+  }
+
   cli::cli_h2("Transformations")
   uc <- prov$transformations$units_conversion
   if (isTRUE(uc$applied)) {
@@ -74,6 +219,16 @@ cog_explain <- function(result, format = c("print", "list")) {
   pc <- prov$transformations$per_capita
   if (isTRUE(pc$applied)) {
     cli::cli_text("Per-capita denominator: {pc$denominator_source}")
+    if (length(pc$popyear_range) == 2L) {
+      lo <- .expand_popyear(pc$popyear_range[1])
+      hi <- .expand_popyear(pc$popyear_range[2])
+      cli::cli_text("  popyear range: {lo}-{hi}")
+    }
+    if (!is.null(pc$pop_source_counts)) {
+      cli::cli_text(
+        "  pop_source counts: census_f33={pc$pop_source_counts$census_f33}, unavailable={pc$pop_source_counts$unavailable}"
+      )
+    }
   }
   infl <- prov$transformations$inflation
   if (isTRUE(infl$applied)) {
@@ -98,3 +253,37 @@ cog_explain <- function(result, format = c("print", "list")) {
 
   invisible(NULL)
 }
+
+# One "break-story" line per referenced break_id: "SB109 (2005): <join_advice>".
+# Re-queries series_breaks_pq for the detail (break_year, join_advice) that
+# provenance$series_break_refs deliberately doesn't carry (the schema keeps
+# that field to a plain id array). Falls back to bare ids if no session is
+# available (e.g. explaining a result after cog_close()) rather than
+# erroring cog_explain() over a cosmetic detail.
+#' @noRd
+.series_break_story_lines <- function(break_ids) {
+  con <- tryCatch(.ensure_session(), error = function(e) NULL)
+  if (is.null(con) || !DBI::dbIsValid(con)) return(break_ids)
+  detail <- tryCatch(
+    DBI::dbGetQuery(con, sprintf(
+      "SELECT break_id, break_year, join_advice FROM series_breaks_pq
+       WHERE break_id IN (%s) ORDER BY break_id",
+      .sql_lit_chr(break_ids)
+    )),
+    error = function(e) NULL
+  )
+  if (is.null(detail) || nrow(detail) == 0L) return(break_ids)
+  sprintf("%s (%s): %s", detail$break_id, detail$break_year, detail$join_advice)
+}
+
+# Expand a 2-digit Census popyear (e.g. 19) to a 4-digit calendar year (2019).
+# F-33 metadata stores popyear as 2 digits; pivot at 70 to handle a future
+# corpus that ever spans pre-1970 vintages, though current scope is 2000+.
+#' @noRd
+.expand_popyear <- function(yy) {
+  yy <- as.integer(yy)
+  if (length(yy) == 0L || is.na(yy)) return(NA_integer_)
+  if (yy >= 100L) return(yy)        # already 4-digit
+  if (yy < 70L)  return(2000L + yy)
+  1900L + yy
+}

R/manifest.R

@@ -1,5 +1,66 @@
 # R/manifest.R
 
+# Sentinel substring baked into the placeholder default URL. If we see this
+# in the resolved URL, the user hasn't configured USCOGDATA_URL yet.
+.PLACEHOLDER_TOKEN <- "REPLACE_WITH_SHARE_TOKEN"
+
+#' Abort with actionable guidance when the resolved corpus URL is still the
+#' placeholder shipped with the package (or any URL containing the sentinel).
+#' Called from `cog_open()` before any I/O so users see a clear message
+#' instead of a downstream JSON parse error.
+#' @noRd
+.check_url_configured <- function(url) {
+  if (!is.character(url) || length(url) != 1L || !nzchar(url)) {
+    cli::cli_abort(c(
+      "USCOGDATA_URL is not configured.",
+      i = "Set the corpus location via one of:",
+      "*" = "{.code Sys.setenv(USCOGDATA_URL = \"<url-or-local-path>/\")}",
+      "*" = "{.code options(uscogdata.url = \"<url-or-local-path>/\")}",
+      i = "For an offline smoke test, use the bundled fixture: {.code system.file(\"extdata/fixture_corpus\", package = \"uscogdata\")}."
+    ), class = "uscogdata_url_not_configured")
+  }
+  if (grepl(.PLACEHOLDER_TOKEN, url, fixed = TRUE)) {
+    sentinel <- .PLACEHOLDER_TOKEN
+    cli::cli_abort(c(
+      "USCOGDATA_URL is not configured (placeholder URL detected).",
+      x = "Current value contains the sentinel {.val {sentinel}}: {.url {url}}",
+      i = "Set the corpus location via one of:",
+      "*" = "{.code Sys.setenv(USCOGDATA_URL = \"<url-or-local-path>/\")}",
+      "*" = "{.code options(uscogdata.url = \"<url-or-local-path>/\")}",
+      i = "For an offline smoke test, use the bundled fixture: {.code system.file(\"extdata/fixture_corpus\", package = \"uscogdata\")}.",
+      i = "For the live Civilytics corpus, request the Nextcloud share URL from the package maintainer."
+    ), class = "uscogdata_url_not_configured")
+  }
+  invisible(url)
+}
+
+#' Try to parse a JSON file. Returns parsed object on success, NULL on
+#' any parse failure (so callers can decide whether to refetch).
+#' @noRd
+.try_parse_manifest_file <- function(path) {
+  tryCatch(
+    jsonlite::fromJSON(path, simplifyVector = FALSE),
+    error = function(e) NULL
+  )
+}
+
+#' Abort with a clear, classified error when a manifest payload (string or
+#' file) cannot be parsed as JSON. Surfaces the URL, content-type if known,
+#' and the underlying parse error.
+#' @noRd
+.abort_invalid_manifest <- function(source, content_type = NA_character_, parse_error = NULL) {
+  ct <- if (is.na(content_type) || !nzchar(content_type)) "<unknown>" else content_type
+  pmsg <- if (is.null(parse_error)) "" else conditionMessage(parse_error)
+  cli::cli_abort(c(
+    "Corpus manifest is not valid JSON.",
+    x = "Source: {source}",
+    i = "Content-Type: {ct}",
+    i = "Likely causes: USCOGDATA_URL points at a login page, a 404 HTML page, or the wrong share; or the corpus has not been published yet.",
+    i = "Set USCOGDATA_URL to a directory (local path or HTTPS) that serves manifest.json directly.",
+    if (nzchar(pmsg)) c(">" = "Parse error: {pmsg}") else NULL
+  ), class = "uscogdata_invalid_manifest")
+}
+
 #' Fetch manifest.json from URL (or read from a local fixture path),
 #' cache locally, validate TTL.
 #' @noRd
@@ -11,23 +72,54 @@
     if (!file.exists(local_manifest)) {
       cli::cli_abort("Local fixture has no manifest.json at {local_manifest}")
     }
-    return(jsonlite::fromJSON(local_manifest, simplifyVector = FALSE))
+    return(tryCatch(
+      jsonlite::fromJSON(local_manifest, simplifyVector = FALSE),
+      error = function(e) .abort_invalid_manifest(source = local_manifest, parse_error = e)
+    ))
   }
 
   cache_path <- file.path(cache_dir, "manifest.json")
   ttl <- as.integer(.cfg("manifest_ttl_secs"))
 
-  needs_fetch <- !file.exists(cache_path) ||
-                 difftime(Sys.time(), file.info(cache_path)$mtime, units = "secs") > ttl
+  cache_fresh <- file.exists(cache_path) &&
+    difftime(Sys.time(), file.info(cache_path)$mtime, units = "secs") <= ttl
+
+  # Honor a fresh cache only if its contents still parse as JSON. A previous
+  # version of this package could write HTML directly into the cache; treat
+  # such poisoned caches as if they were missing so the next call recovers.
+  if (cache_fresh) {
+    parsed <- .try_parse_manifest_file(cache_path)
+    if (!is.null(parsed)) return(parsed)
+  }
 
-  if (needs_fetch) {
   resp <- httr2::request(paste0(url, "manifest.json")) |>
     httr2::req_error(is_error = function(r) httr2::resp_status(r) >= 400) |>
     httr2::req_perform()
-    writeLines(httr2::resp_body_string(resp), cache_path)
-  }
+  body <- httr2::resp_body_string(resp)
 
-  jsonlite::fromJSON(cache_path, simplifyVector = FALSE)
+  # Parse BEFORE persisting. If the server returned HTML / a login page /
+  # any non-JSON body with a 2xx status, we must not write it to the cache.
+  parsed <- tryCatch(
+    jsonlite::fromJSON(body, simplifyVector = FALSE),
+    error = function(e) {
+      ct <- tryCatch(httr2::resp_content_type(resp), error = function(e2) NA_character_)
+      .abort_invalid_manifest(
+        source = paste0(url, "manifest.json"),
+        content_type = ct,
+        parse_error = e
+      )
+    }
+  )
+
+  # Atomic write: tmp file alongside cache_path (same filesystem -> no EXDEV)
+  # then rename. Ensures a partial write or interrupted process never
+  # replaces a previously-good cache.
+  if (!dir.exists(cache_dir)) dir.create(cache_dir, recursive = TRUE)
+  tmp <- paste0(cache_path, ".tmp.", Sys.getpid())
+  on.exit(if (file.exists(tmp)) unlink(tmp), add = TRUE)
+  writeLines(body, tmp)
+  file.rename(tmp, cache_path)
+  parsed
 }
 
 #' @noRd
@@ -36,11 +128,21 @@
 }
 
 #' @noRd
-.validate_schema <- function(manifest, expected_version) {
-  if (manifest$schema_version != expected_version) {
+#' Schema v6 (FIPS geography harmonization, 2026-07-22) is accepted alongside
+#' 4/5. v6 renamed the long table's fips_state_code/fips_county_code to
+#' fips_state_asof/fips_county_asof and added cog_legacy_state/
+#' cog_legacy_county (26 -> 28 cols); this package references NONE of those
+#' columns, so no code change was needed. NOTE the SILENT semantic change for
+#' any consumer of the raw long table: long fips_state/fips_county are now
+#' PRESENT/harmonized geography (current county identity carried back to every
+#' year, matching canonical_fips_xwalk) rather than as-of-year; as-of-year
+#' moved to the *_asof columns. This package's own geography always came from
+#' the xwalk (already present-based), so behaviour is unchanged.
+.validate_schema <- function(manifest, supported = c(4L, 5L, 6L, 7L)) {
+  if (!manifest$schema_version %in% supported) {
     cli::cli_abort(c(
       "Corpus schema version mismatch.",
-      x = "Package expects schema_version = {expected_version}; corpus has {manifest$schema_version}.",
+      x = "Package supports schema_version in {paste(supported, collapse = ', ')}; corpus has {manifest$schema_version}.",
       i = "Update uscogdata (install.packages or pak::pkg_install) or re-publish corpus."
     ))
   }
@@ -54,3 +156,18 @@
 }
 
 `%||%` <- function(a, b) if (is.null(a) || (length(a) == 1 && is.na(a))) b else a
+
+#' Return the parsed corpus manifest for the active session.
+#'
+#' Opens a session (connecting to the configured corpus) if none is active,
+#' then returns the manifest exactly as parsed from `manifest.json`. Useful
+#' for consumers that need the published year range (`years` block, schema
+#' v5+) or the partition list without issuing a data query.
+#'
+#' @return Named list: `schema_version`, `built_at`, `pipeline_commit`,
+#'   `data_vintage`, `scope`, `years` (schema v5+), `schema`, `files`.
+#' @export
+cog_manifest <- function() {
+  .ensure_session()
+  .uscogdata_env$manifest
+}

R/peers.R

@@ -2,32 +2,43 @@
 
 #' Find peer governments by similarity criteria
 #'
-#' Selects peer governments from `canonical_fips_xwalk` by combinations of
-#' government type, state, and population range. Peers are ordered by
-#' `|log(pop_ratio)|` ascending (closest to the target's population first).
+#' Selects peer governments by combinations of government type, state, and
+#' population range at a chosen `year`. Peers are ordered by `|log(pop_ratio)|`
+#' ascending (closest to the target's population first).
 #'
 #' @param target_govid Character scalar — `canonical_govid` of the target.
+#' @param year Integer scalar. Cohort vintage. When `NULL` (default), uses the
+#'   most recent year for which the target has an observed population in
+#'   `gov_population_yearly`.
 #' @param same_type If `TRUE` (default) restrict peers to the target's
 #'   `govs_type`.
 #' @param same_state If `TRUE` restrict peers to the target's `fips_state`.
 #'   Default `FALSE`.
 #' @param pop_range Length-2 numeric vector giving lower/upper bounds.
 #' @param is_ratio If `TRUE` (default) `pop_range` is multiplied by the
-#'   target's `population_acs` to produce absolute bounds. If `FALSE`,
+#'   target's population at `year` to produce absolute bounds. If `FALSE`,
 #'   `pop_range` is interpreted as absolute population counts.
-#' @param pop_year Reserved for future use (selecting ACS vintage). Currently
-#'   the corpus has a single snapshot so this argument has no effect.
 #' @param max_peers Integer cap on the number of peers returned.
+#' @param coverage Survey-cycle handling; see [cog_peer_compare()]. Here it
+#'   governs the cohort VINTAGE when `year` is `NULL`: `"census"` snaps to the
+#'   most recent census year with an observed population, so a cohort is not
+#'   built from a sample year in which most of the candidate universe is
+#'   absent. `"consistent"` needs a year range, which cohort selection does not
+#'   have, so it selects like `"all"` and is carried on the result as
+#'   `attr(x, "coverage")` for [cog_peer_compare()].
 #' @return Tibble with columns `canonical_govid`, `gov_name`, `fips_state`,
-#'   `population_acs`, `pop_ratio`, `rank`.
+#'   `population`, `pop_ratio`, `rank`. The cohort year is attached as
+#'   `attr(x, "cohort_year")`.
 #' @export
 cog_find_peers <- function(target_govid,
+                           year = NULL,
                            same_type = TRUE,
                            same_state = FALSE,
                            pop_range = c(0.7, 1.3),
                            is_ratio = TRUE,
-                           pop_year = NULL,
-                           max_peers = 10L) {
+                           max_peers = 10L,
+                           coverage = c("all", "census", "consistent")) {
+  coverage <- .validate_coverage(coverage)
   if (!is.character(target_govid) || length(target_govid) != 1L) {
     cli::cli_abort("`target_govid` must be a length-1 character string.")
   }
@@ -35,65 +46,127 @@ cog_find_peers <- function(target_govid,
       pop_range[1] >= pop_range[2]) {
     cli::cli_abort("`pop_range` must be a length-2 numeric with lo < hi.")
   }
+  if (!is.null(year) &&
+      (!(is.numeric(year) || is.integer(year)) || length(year) != 1L)) {
+    cli::cli_abort("`year` must be NULL or a length-1 integer.")
+  }
 
   con <- .ensure_session()
 
-  target_sql <- sprintf(
-    "SELECT canonical_govid, gov_name, govs_type, fips_state, population_acs
+  # Confirm target exists in the xwalk and pull govs_type / fips_state.
+  meta_sql <- sprintf(
+    "SELECT canonical_govid, gov_name, govs_type, fips_state
      FROM canonical_fips_xwalk
      WHERE canonical_govid = %s",
     .sql_lit_chr(target_govid)
   )
-  target <- DBI::dbGetQuery(con, target_sql)
-  if (nrow(target) == 0L) {
+  meta <- DBI::dbGetQuery(con, meta_sql)
+  if (nrow(meta) == 0L) {
     cli::cli_abort(c(
       "govid {target_govid} not found in corpus.",
       i = "v0.1 covers types 0-3 only (state/county/city/township); see vignette('coverage-scope')."
     ))
   }
-  if (is.na(target$population_acs) || target$population_acs <= 0) {
-    cli::cli_abort("Target {target_govid} has missing or non-positive population; cannot build pop_ratio band.")
+
+  cohort_year <- .resolve_cohort_year(con, target_govid, year, coverage)
+
+  pop_sql <- sprintf(
+    "SELECT population FROM gov_population_yearly
+     WHERE canonical_govid = %s AND year = %d",
+    .sql_lit_chr(target_govid), as.integer(cohort_year)
+  )
+  target_pop <- DBI::dbGetQuery(con, pop_sql)$population
+  if (length(target_pop) == 0L || is.na(target_pop) || target_pop <= 0) {
+    cli::cli_abort(c(
+      "Target {target_govid} has no observed population in {cohort_year}.",
+      i = "Use a year for which population is observed; see gov_population_yearly."
+    ))
   }
 
   if (isTRUE(is_ratio)) {
-    lo <- target$population_acs * pop_range[1]
-    hi <- target$population_acs * pop_range[2]
+    lo <- target_pop * pop_range[1]
+    hi <- target_pop * pop_range[2]
   } else {
     lo <- pop_range[1]; hi <- pop_range[2]
   }
 
   preds <- c(
-    sprintf("canonical_govid != %s", .sql_lit_chr(target_govid)),
-    sprintf("population_acs BETWEEN %.6f AND %.6f", lo, hi)
+    sprintf("p.canonical_govid != %s", .sql_lit_chr(target_govid)),
+    sprintf("p.year = %d", as.integer(cohort_year)),
+    sprintf("p.population BETWEEN %.6f AND %.6f", lo, hi)
   )
-  if (isTRUE(same_type))  preds <- c(preds, sprintf("govs_type = %d", target$govs_type))
-  if (isTRUE(same_state)) preds <- c(preds, sprintf("fips_state = %s", .sql_lit_chr(target$fips_state)))
+  if (isTRUE(same_type))  preds <- c(preds, sprintf("x.govs_type = %d", meta$govs_type))
+  if (isTRUE(same_state)) preds <- c(preds, sprintf("x.fips_state = %s", .sql_lit_chr(meta$fips_state)))
 
   peers_sql <- sprintf(
-    "SELECT canonical_govid, gov_name, fips_state, population_acs,
-            population_acs / %.6f AS pop_ratio
-     FROM canonical_fips_xwalk
+    "SELECT p.canonical_govid, x.gov_name, x.fips_state, p.population,
+            p.population / %.6f AS pop_ratio
+     FROM gov_population_yearly p
+     JOIN canonical_fips_xwalk x USING (canonical_govid)
      WHERE %s
-     ORDER BY ABS(LN(CAST(population_acs AS DOUBLE) / %.6f))
+     ORDER BY ABS(LN(CAST(p.population AS DOUBLE) / %.6f))
      LIMIT %d",
-    target$population_acs,
+    target_pop,
     paste(preds, collapse = " AND "),
-    target$population_acs,
+    target_pop,
     as.integer(max_peers)
   )
   peers <- tibble::as_tibble(DBI::dbGetQuery(con, peers_sql))
-  if (nrow(peers) > 0L) peers$rank <- seq_len(nrow(peers))
-  else peers$rank <- integer(0)
+  peers$rank <- if (nrow(peers) > 0L) seq_len(nrow(peers)) else integer(0)
+  attr(peers, "cohort_year") <- as.integer(cohort_year)
+  attr(peers, "pop_range")   <- as.numeric(pop_range)
+  attr(peers, "is_ratio")    <- isTRUE(is_ratio)
+  attr(peers, "coverage")    <- coverage
+  attr(peers, "is_census_year") <- .is_census_year(cohort_year)
   peers
 }
 
+# `coverage` picks the cohort vintage when the caller did not name one.
+# "census" snaps to the most recent CENSUS year with an observed population,
+# so a cohort is not silently built from a sample year in which most of the
+# candidate universe is absent. "consistent" is a comparison-time concept --
+# it needs a year RANGE, which cohort selection does not have -- so it selects
+# like "all" here and is carried on the result for cog_peer_compare().
+#' @noRd
+.resolve_cohort_year <- function(con, target_govid, year,
+                                 coverage = "all") {
+  if (!is.null(year)) return(as.integer(year))
+  if (identical(coverage, "census")) {
+    sql <- sprintf(
+      "SELECT MAX(year) AS y FROM gov_population_yearly
+       WHERE canonical_govid = %s AND year %% 10 IN (2, 7)",
+      .sql_lit_chr(target_govid)
+    )
+    y <- DBI::dbGetQuery(con, sql)$y
+    if (length(y) > 0L && !is.na(y)) return(as.integer(y))
+    cli::cli_abort(c(
+      "{.code coverage = \"census\"} found no census year with an observed population for {target_govid}.",
+      i = "Pass an explicit {.arg year}, or use {.code coverage = \"all\"}."
+    ), class = "uscogdata_no_census_years")
+  }
+  sql <- sprintf(
+    "SELECT MAX(year) AS y FROM gov_population_yearly
+     WHERE canonical_govid = %s",
+    .sql_lit_chr(target_govid)
+  )
+  y <- DBI::dbGetQuery(con, sql)$y
+  if (length(y) == 0L || is.na(y)) {
+    cli::cli_abort(
+      "Target {target_govid} has no observed population in any year."
+    )
+  }
+  as.integer(y)
+}
+
 #' Compare a target government against a peer set
 #'
 #' Pulls spending for the target plus a peer set (either a
 #' [cog_find_peers()] result or a character vector of `canonical_govid`) and
 #' appends peer-distribution summary rows (`summary_p25`, `summary_p50`,
 #' `summary_p75`) so the result can be faceted by `role` in a single ggplot
-#' call.
+#' call. Those summary rows are quantiles **within each category**, not
+#' quantiles of each peer's total — see the `@return` section before summing
+#' them.
 #'
 #' @param target_govid Character scalar.
 #' @param peers A tibble from [cog_find_peers()] or a character vector of
@@ -103,18 +176,92 @@ cog_find_peers <- function(target_govid,
 #' @param per_capita Default `TRUE` — peer compare usually normalizes by
 #'   population.
 #' @param adjust_to_year Integer base year for CPI-U conversion or `NULL`.
+#' @param expenditure_concept `"primary"` (default), `"direct"`, or
+#'   `"total"` -- see [cog_spending()] for the three concepts. `"total"` is
+#'   refused here because combining Total across peer sets counts
+#'   intergovernmental transfers twice; `"primary"` and `"direct"` combine
+#'   safely.
+#' @param coverage How to handle the Census of Governments survey cycle,
+#'   which is a **complete census only in years ending in 2 and 7** -- every
+#'   other year is a sample, and the sample varies enormously (on the bundled
+#'   fixture, Wisconsin's 608-city universe reports 597 governments in FY2012
+#'   and 112 in FY2019).
+#'
+#'   * `"all"` (default) -- every unit that reported that year. Unchanged
+#'     behaviour, so existing code keeps working.
+#'   * `"census"` -- census years only. Aborts if the requested range holds
+#'     none, rather than silently returning nothing.
+#'   * `"consistent"` -- only units reporting in *every* requested year, giving
+#'     a balanced panel.
+#'
+#'   Regardless of mode, `provenance$coverage` always carries per-year
+#'   `n_units_reporting`, `n_units_expected` and `is_census_year`, and
+#'   `provenance$coverage_mode` records the mode. `is_census_year` is a
+#'   statement about the **survey calendar**, never a claim of completeness:
+#'   FY1967 is a census year in which only 97 of Wisconsin's 608 cities
+#'   report. `n_units_reporting` is the number that tells the truth.
+#'
+#'   The comparison target is exempt from `"consistent"` balancing -- it is the
+#'   subject of the comparison, not a member of the cohort -- and the
+#'   `summary_*` quantiles are computed AFTER the filter, so they describe the
+#'   cohort actually returned. `n_units_reporting` counts peers only, against
+#'   the cohort size: "3 of your 15 peers reported in FY2019".
 #' @return Tibble matching [cog_spending()]'s columns, plus a `role`
 #'   column taking values `"target"`, `"peer"`, `"summary_p25"`,
-#'   `"summary_p50"`, or `"summary_p75"`, and `target_rank` (target's rank
-#'   among target+peers at `max(years)`, NA for other rows). Provenance
-#'   attribute reports `verb = "cog_peer_compare"` and `peer_count`.
+#'   `"summary_p50"`, or `"summary_p75"`, `target_rank` (target's rank
+#'   among target+peers at `max(years)`, NA for other rows), and
+#'   `cohort_year` (the year used to build the peer cohort, read from
+#'   `attr(peers, "cohort_year")`; `NA` when `peers` was a bare character
+#'   vector). Provenance reports `verb = "cog_peer_compare"`, `peer_count`,
+#'   `cohort_year`, and `cohort_govids`.
+#'
+#'   **The `summary_*` rows are per-category quantiles: they are not additive.**
+#'   Each one is computed **within each `(year, spend_subtype,
+#'   category)` cell** across the peer set, so a `summary_p50` row is *the
+#'   median peer's value in that one category*, not *the value of the median
+#'   peer's total*. The median peer for Police and the median peer for Fire
+#'   are usually different governments, so summing `summary_*` rows across
+#'   categories does not give any peer's total and misstates the band it
+#'   appears to describe — measured at −32.7% to +251.0% across 24 years on
+#'   one cohort, with a sign flip at FY2012.
+#'
+#'   Facet by `role` **and** `category` (the documented use, and what the
+#'   rows are built for). For a genuine "median peer's total spending" line,
+#'   sum each peer's own categories first and take the quantile of those
+#'   per-government totals:
+#'
+#'   ```r
+#'   library(dplyr)
+#'   cmp |>
+#'     filter(role %in% c("target", "peer")) |>
+#'     group_by(year, role, canonical_govid) |>
+#'     summarise(total = sum(amt_per_capita_real, na.rm = TRUE), .groups = "drop") |>
+#'     filter(role == "peer") |>
+#'     group_by(year) |>
+#'     summarise(p50 = quantile(total, 0.5, na.rm = TRUE))
+#'   ```
 #' @export
 cog_peer_compare <- function(target_govid, peers, category, years,
-                             per_capita = TRUE, adjust_to_year = NULL) {
+                             per_capita = TRUE, adjust_to_year = NULL,
+                             expenditure_concept = c("primary", "direct", "total"),
+                             coverage = c("all", "census", "consistent")) {
   call <- match.call()
+  expenditure_concept <- match.arg(expenditure_concept)
+  coverage <- .validate_coverage(coverage)
+  if (identical(expenditure_concept, "total")) {
+    .abort_concept_not_aggregatable("cog_peer_compare")
+  }
   if (!is.character(target_govid) || length(target_govid) != 1L) {
     cli::cli_abort("`target_govid` must be a length-1 character string.")
   }
+  cohort_year <- if (is.data.frame(peers)) {
+    ay <- attr(peers, "cohort_year")
+    if (is.null(ay)) NA_integer_ else as.integer(ay)
+  } else {
+    NA_integer_
+  }
+  pop_range <- if (is.data.frame(peers)) attr(peers, "pop_range") else NULL
+  is_ratio  <- if (is.data.frame(peers)) attr(peers, "is_ratio")  else NULL
   peer_govids <- if (is.data.frame(peers)) {
     as.character(peers$canonical_govid)
   } else {
@@ -123,24 +270,49 @@ cog_peer_compare <- function(target_govid, peers, category, years,
   peer_govids <- peer_govids[!is.na(peer_govids) & nzchar(peer_govids)]
   all_govids <- unique(c(target_govid, peer_govids))
 
-  r <- cog_spending(all_govids, years, category, per_capita, adjust_to_year)
+  years <- .apply_census_years(years, coverage, "cog_peer_compare")
+
+  r <- cog_spending(all_govids, years, category, per_capita, adjust_to_year,
+                    expenditure_concept = expenditure_concept)
   r$role <- ifelse(r$canonical_govid == target_govid, "target", "peer")
 
+  # The target is exempt from balancing: it is the subject of the comparison,
+  # not a member of the cohort being balanced, and dropping it would leave a
+  # peer comparison with nothing to compare. Filtering happens BEFORE the
+  # quantiles below, so a "consistent" cohort's summary rows describe that
+  # cohort rather than the unbalanced one.
+  if (identical(coverage, "consistent")) {
+    r <- .filter_consistent(r, years, keep_ids = target_govid)
+  }
+
   value_col <- .peer_value_col(per_capita, adjust_to_year)
 
   summary_rows <- .peer_summary_rows(r, value_col)
   out <- dplyr::bind_rows(r, summary_rows)
   rank_val <- .peer_target_rank(r, target_govid, years, value_col)
   out$target_rank <- ifelse(out$role == "target", rank_val, NA_integer_)
+  out$cohort_year <- cohort_year
 
   prov <- attr(r, "provenance") %||% list()
   prov$verb        <- "cog_peer_compare"
   prov$call        <- paste(deparse(call), collapse = " ")
   prov$peer_count  <- length(peer_govids)
+  prov$cohort_year <- cohort_year
+  prov$cohort_govids <- peer_govids
+  prov$pop_range   <- pop_range
+  prov$is_ratio    <- is_ratio
   prov$target     <- list(
     canonical_govid = target_govid,
     gov_name        = unique(r$gov_name[r$role == "target"])
   )
+  # Counted over PEER rows only, against the cohort size: "3 of your 15 peers
+  # reported in FY2019". Including the target would inflate every count by one
+  # and make a cohort that has entirely stopped reporting look non-empty.
+  prov$coverage_mode <- coverage
+  prov$coverage <- .coverage_table(
+    out, years, length(peer_govids),
+    rows = r[r$role == "peer", , drop = FALSE]
+  )
   attr(out, "provenance") <- prov
   out
 }

R/provenance.R

@@ -4,7 +4,15 @@
 #' @noRd
 .build_provenance <- function(verb, call, govid, years, category,
                               per_capita, adjust_to_year, result, sql,
-                              subtype_col) {
+                              subtype_col, basis = NA_character_,
+                              basis_note = NA_character_,
+                              expenditure_concept = "primary",
+                              expenditure_concept_note = NA_character_,
+                              expenditure_concept_direct_suppressed = FALSE,
+                              revenue_concept = "general",
+                              harmonization = NULL, recipe = NULL,
+                              suggestions = list(),
+                              completion = NULL) {
   manifest <- .uscogdata_env$manifest
 
   codes <- result[["codes_included"]]
@@ -29,6 +37,23 @@
     unique(result$gov_name)
   }
 
+  schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
+  con <- .uscogdata_env$con
+  have_con <- !is.null(con) && DBI::dbIsValid(con)
+  break_refs <- if (have_con) {
+    .build_series_break_refs(con, codes_observed, years, schema_version)
+  } else {
+    character(0)
+  }
+  # Corpus-wide caveats travel separately: they qualify the whole result
+  # rather than one series, and they do not depend on codes_observed (see
+  # .build_corpus_break_refs()).
+  corpus_refs <- if (have_con) {
+    .build_corpus_break_refs(con, years, schema_version)
+  } else {
+    character(0)
+  }
+
   list(
     verb = verb,
     call = paste(deparse(call), collapse = " "),
@@ -38,6 +63,18 @@
     ),
     years = as.integer(years),
     category = category,
+    basis = basis,
+    basis_note = basis_note,
+    expenditure_concept = expenditure_concept,
+    expenditure_concept_note = expenditure_concept_note,
+    expenditure_concept_direct_suppressed = isTRUE(expenditure_concept_direct_suppressed),
+    revenue_concept = revenue_concept,
+    harmonization = harmonization %||% list(
+      applied = FALSE, na_rows_excluded = 0L, na_amount_excluded = 0,
+      note = NA_character_
+    ),
+    recipe = recipe,
+    suggestions = suggestions,
     scope = list(
       gov_types_included = as.integer(unlist(manifest$scope$gov_types_included)),
       gov_types_excluded = as.integer(unlist(manifest$scope$gov_types_excluded)),
@@ -61,9 +98,27 @@
       per_capita = list(
         applied = isTRUE(per_capita),
         denominator_source = if (isTRUE(per_capita)) {
-          "ACS 2018-2022 B01003_001 (population_acs from canonical_fips_xwalk)"
+          "Census F-33 population (per-year, from long.population)"
         } else {
           NA_character_
+        },
+        popyear_range = if (isTRUE(per_capita)) {
+          attr(result, ".popyear_range") %||% integer(0)
+        } else {
+          integer(0)
+        },
+        pop_source_counts = if (isTRUE(per_capita)) {
+          ps <- result[["pop_source"]]
+          if (is.null(ps) || length(ps) == 0L) {
+            list(census_f33 = 0L, unavailable = 0L)
+          } else {
+            list(
+              census_f33  = sum(ps == "census_f33", na.rm = TRUE),
+              unavailable = sum(ps == "unavailable", na.rm = TRUE)
+            )
+          }
+        } else {
+          NULL
         }
       ),
       inflation = list(
@@ -72,7 +127,15 @@
         index = if (is.null(adjust_to_year)) NA_character_ else "CPI-U (BLS CPIAUCSL annual average, bundled)"
       )
     ),
-    series_break_refs = character(0),
+    series_break_refs = break_refs,
+    corpus_break_refs = corpus_refs,
+    # What `complete = TRUE` filled, and the rule it filled by. Always
+    # present so a consumer can read `completion$applied` without testing
+    # for the key -- an absent block and applied = FALSE would otherwise be
+    # indistinguishable from an older reader version.
+    completion = completion %||% list(
+      applied = FALSE, rows_filled = 0L, absence_means = list()
+    ),
     manifest = list(
       schema_version = as.integer(manifest$schema_version),
       pipeline_commit = manifest$pipeline_commit %||% NA_character_,

R/recipes.R

@@ -0,0 +1,167 @@
+# R/recipes.R
+# Harmonization recipes: multi-code, cross-vintage series built by summing a
+# fixed set of component item codes with per-component weights and
+# year/gov-type scoping (see the `harmonization_recipes` view, registered
+# from data/harmonization_recipes.parquet, schema_version >= 5 only).
+#
+# Recipes exist because some cross-vintage series can't be expressed as a
+# 1:1 harmonized_code mapping (basis = "harmonized"): the wide era (pre-2012)
+# publishes only a combined aggregate row for these families (e.g.
+# corrections functions 04+05), while the modern era splits them into leaf
+# codes. A recipe's generic join sums whichever of its component codes are
+# present for a given year, so the resulting series is continuous across
+# that format boundary.
+
+#' List available harmonization recipes
+#'
+#' Recipes are multi-code cross-vintage series (see [cog_spending()]'s
+#' `recipe` argument) catalogued in the corpus's `harmonization_recipes`
+#' table. Use this to discover valid `recipe` ids.
+#'
+#' @param pattern Optional regex matched case-insensitively against
+#'   `recipe_id` or `label`.
+#' @return Tibble with columns `recipe_id`, `label`, `n_components`,
+#'   `year_min`, `year_max` (the min/max component year coverage), sorted by
+#'   `recipe_id`.
+#' @export
+cog_recipes <- function(pattern = NULL) {
+  if (!is.null(pattern) &&
+      (!is.character(pattern) || length(pattern) != 1L)) {
+    cli::cli_abort("`pattern` must be a length-1 character string or NULL.")
+  }
+  con <- .ensure_session()
+  .require_schema_v5(con, .uscogdata_env$manifest, "cog_recipes()")
+
+  where <- if (is.null(pattern)) {
+    ""
+  } else {
+    sprintf(
+      "WHERE regexp_matches(recipe_id, %1$s, 'i') OR regexp_matches(label, %1$s, 'i')",
+      .sql_lit_chr(pattern)
+    )
+  }
+  sql <- paste(
+    "SELECT recipe_id, any_value(label) AS label,
+            COUNT(*) AS n_components,
+            MIN(year_min) AS year_min, MAX(year_max) AS year_max
+     FROM harmonization_recipes",
+    where,
+    "GROUP BY recipe_id
+     ORDER BY recipe_id"
+  )
+  out <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
+  out$year_min <- as.integer(out$year_min)
+  out$year_max <- as.integer(out$year_max)
+  out$n_components <- as.integer(out$n_components)
+  out
+}
+
+#' Abort unless the active corpus has schema_version >= 5.
+#' @noRd
+.require_schema_v5 <- function(con, manifest, what) {
+  sv <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
+  if (sv < 5L) {
+    cli::cli_abort(c(
+      sprintf("%s requires corpus schema_version >= 5.", what),
+      x = "Active corpus has schema_version {sv}.",
+      i = "Point USCOGDATA_URL at a schema_version >= 5 corpus to use harmonization recipes."
+    ), class = "uscogdata_schema_unsupported")
+  }
+  invisible(sv)
+}
+
+#' Abort with the valid id list unless `recipe_id` exists in the catalog.
+#' @noRd
+.validate_recipe_id <- function(con, recipe_id) {
+  ids <- DBI::dbGetQuery(
+    con, "SELECT DISTINCT recipe_id FROM harmonization_recipes"
+  )$recipe_id
+  if (!recipe_id %in% ids) {
+    cli::cli_abort(c(
+      "Unknown recipe = {.val {recipe_id}}.",
+      i = "Valid ids: {paste(sort(ids), collapse = ', ')}",
+      i = "See cog_recipes() for labels and year coverage."
+    ), class = "uscogdata_unknown_recipe")
+  }
+  invisible(TRUE)
+}
+
+#' Fetch the component rows for one recipe (label, component codes, scope,
+#' year ranges, weights) -- both for running the recipe and for the
+#' `recipe` provenance block.
+#' @noRd
+.recipe_components <- function(con, recipe_id) {
+  sql <- sprintf(
+    "SELECT recipe_id, label, component_code, gov_type_scope,
+            year_min, year_max, weight, source_break_ids, notes
+     FROM harmonization_recipes
+     WHERE recipe_id = %s
+     ORDER BY component_code",
+    .sql_lit_chr(recipe_id)
+  )
+  tibble::as_tibble(DBI::dbGetQuery(con, sql))
+}
+
+#' Run a recipe's generic join: sum `amt * weight` across whichever
+#' component codes are present for each (year, canonical_govid), scoped by
+#' gov_type_scope. Deliberately does NOT filter `NOT is_aggregate`: in the
+#' wide era (<= 2011) these families' component codes exist ONLY as
+#' aggregate rows (leaves first appear 2012), so excluding aggregates would
+#' zero out the wide-era half of every recipe. This is safe by corpus
+#' construction -- wide-era rows for these codes are aggregate-only, modern
+#' rows are leaf-only, and every component row is year-scoped via
+#' `year_min`/`year_max` -- so there is no double-counting. (Checkpoint
+#' review docs/phase_r_harmonization_review.md § 0.2.)
+#' @noRd
+.run_recipe <- function(con, recipe_id, govid, years) {
+  sql <- sprintf(
+    "SELECT l.year, l.canonical_govid,
+            COALESCE(x.gov_name, l.gov_name) AS gov_name,
+            SUM(l.amt * r.weight) * 1000.0 AS amt_nominal,
+            string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code) AS codes_included
+     FROM long l
+     JOIN harmonization_recipes r
+       ON l.item_code = r.component_code
+      AND l.year BETWEEN r.year_min AND r.year_max
+      AND (r.gov_type_scope = 'all'
+           OR (r.gov_type_scope = 'state' AND l.type = 0)
+           OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
+     LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+     WHERE r.recipe_id = %1$s
+       AND l.canonical_govid IN (%2$s)
+       AND l.year IN (%3$s)
+     GROUP BY 1, 2, 3
+     ORDER BY 1, 2",
+    .sql_lit_chr(recipe_id), .sql_lit_chr(govid),
+    paste(as.integer(years), collapse = ",")
+  )
+  result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
+  attr(result, "sql_query") <- sql
+  result
+}
+
+#' Shape a raw .run_recipe() result into the standard cog_spending()/
+#' cog_revenue() column layout: subtype = "recipe", category = the recipe's
+#' label, aggregate_fallback = FALSE (recipes resolve coverage gaps by
+#' construction, not by falling back to an aggregate row).
+#' @noRd
+.shape_recipe_result <- function(result, subtype_col, label) {
+  sql_query <- attr(result, "sql_query")
+  n <- nrow(result)
+  result[[subtype_col]] <- rep("recipe", n)
+  result$category <- rep(label, n)
+  result$aggregate_fallback <- rep(FALSE, n)
+  result <- result[, c(
+    "year", "canonical_govid", "gov_name", subtype_col, "category",
+    "amt_nominal", "codes_included", "aggregate_fallback"
+  ), drop = FALSE]
+  attr(result, "sql_query") <- sql_query
+  result
+}
+
+#' Turn a small data.frame into a list-of-lists (one list per row), the
+#' shape used for the `recipe$components` provenance block.
+#' @noRd
+.df_to_row_list <- function(df) {
+  lapply(seq_len(nrow(df)), function(i) as.list(df[i, , drop = FALSE]))
+}

R/revenue.R

@@ -8,22 +8,60 @@
 #' multiplies by 1000 and records the conversion in `provenance`).
 #'
 #' @inheritParams cog_spending
+#' @param revenue_concept Which of Census's two published revenue concepts to
+#'   return. Concepts are defined as sets of the crosswalk's `revenue_subtype`
+#'   values -- never as item-code first letters, which cannot classify
+#'   correctly (prefix `Y` spans revenue, expenditure and balance codes, and
+#'   prefix `X` does the same):
+#'
+#'   * `"general"` (default) -- Census General Revenue: `own_source` +
+#'     `federal` + `state` + `local_aid`. The manual defines this concept by
+#'     subtraction (section 4.3: *"General revenue comprises all revenue
+#'     except that classified as liquor store, utility, or insurance trust
+#'     revenue"*), so utility (`A91`-`A94`), liquor store (`A90`) and
+#'     insurance trust revenue are all excluded.
+#'   * `"total"` -- Census Total Revenue: every revenue subtype, i.e.
+#'     `general` plus utility, liquor store, and insurance trust revenue
+#'     (`Y01`/`Y02`/`Y04`/`Y11`/`Y12`/`Y51`/`Y52` and the employee-retirement
+#'     `X01`/`X02`/`X05`/`X08`).
+#'
+#'   The two are related by Census's own identity, `Total Revenue = General +
+#'   Utility + Liquor Store + Insurance Trust`.
+#'
+#'   Note that the employee-retirement (`X`) codes stop at FY2016, when those
+#'   systems moved out of the annual finance file into the separate Annual
+#'   Survey of Public Pensions, so a `"total"` series steps down at the
+#'   FY2016/FY2017 seam for reasons that are about collection scope rather
+#'   than revenue (series breaks `SB197`-`SB202`).
 #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
 #'   `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`,
 #'   optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
-#'   `codes_included`, `aggregate_fallback`, `notes`.
+#'   optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`,
+#'   and `value_source` when `complete = TRUE`.
 #' @export
 cog_revenue <- function(govid, years, category = NULL,
-                        per_capita = FALSE, adjust_to_year = NULL) {
+                        per_capita = FALSE, adjust_to_year = NULL,
+                        basis = c("harmonized", "raw"), recipe = NULL,
+                        revenue_concept = c("general", "total"),
+                        complete = FALSE) {
+  # flow_prefixes no longer classifies rows (crosswalk revenue_subtype
+  # membership does -- General Revenue, i.e. everything except
+  # insurance_trust) -- it only scopes the recipe-suggestion machinery to
+  # this verb's recipe families (see R/suggestions.R).
   .verb_spendrev(
     verb           = "cog_revenue",
-    view           = "revenue_annotated",
+    view_base      = "revenue_annotated",
     subtype_col    = "revenue_subtype",
+    flow_prefixes  = c("T", "A", "U", "B", "C", "D"),
     call           = match.call(),
     govid          = govid,
     years          = years,
     category       = category,
     per_capita     = per_capita,
-    adjust_to_year = adjust_to_year
+    adjust_to_year = adjust_to_year,
+    basis          = basis,
+    recipe         = recipe,
+    revenue_concept = revenue_concept,
+    complete       = complete
   )
 }

R/rollup.R

@@ -8,28 +8,67 @@
 #' "place portraits" that compare a city to the surrounding county and
 #' containing state on one set of axes.
 #'
+#' When `per_capita = TRUE`, rows whose government has no observed
+#' population in that year (`pop_source == "unavailable"`) are dropped from
+#' the result. The dropped govids are recorded in
+#' `provenance$rollup$excluded_govids`. This excludes special districts
+#' (gov type 4) and school districts (gov type 5) from per-capita rollups
+#' by design — see `vignette('population-denominators')`.
+#'
 #' @param govids Named list with any non-empty subset of elements named
 #'   `state`, `county`, `city`. Each element is a character vector of
 #'   `canonical_govid` values. At least one layer required.
 #' @param category Single category name or character vector (passed through
 #'   to [cog_spending()]).
 #' @param years Integer vector of years.
-#' @param per_capita If `TRUE`, per-capita uses each layer's own population
-#'   from `canonical_fips_xwalk.population_acs`.
+#' @param per_capita If `TRUE`, per-capita uses each gov's own per-year
+#'   population from `gov_population_yearly`. Govs with missing population
+#'   are excluded from the result.
 #' @param adjust_to_year Integer base year for CPI-U conversion, or `NULL`.
+#' @param expenditure_concept `"primary"` (default), `"direct"`, or
+#'   `"total"` -- see [cog_spending()] for the three concepts. `"total"` is
+#'   refused here because combining Total across multiple layers of
+#'   government double-counts intergovernmental transfers (a state's payment
+#'   to a school district is the same dollar the district reports as its own
+#'   Direct spending); `"primary"` and `"direct"` combine safely.
+#' @param coverage How to handle the Census of Governments survey cycle,
+#'   which is a **complete census only in years ending in 2 and 7** -- every
+#'   other year is a sample, and the sample varies enormously (on the bundled
+#'   fixture, Wisconsin's 608-city universe reports 597 governments in FY2012
+#'   and 112 in FY2019).
+#'
+#'   * `"all"` (default) -- every unit that reported that year. Unchanged
+#'     behaviour, so existing code keeps working.
+#'   * `"census"` -- census years only. Aborts if the requested range holds
+#'     none, rather than silently returning nothing.
+#'   * `"consistent"` -- only units reporting in *every* requested year, giving
+#'     a balanced panel.
+#'
+#'   Regardless of mode, `provenance$coverage` always carries per-year
+#'   `n_units_reporting`, `n_units_expected` and `is_census_year`, and
+#'   `provenance$coverage_mode` records the mode. `is_census_year` is a
+#'   statement about the **survey calendar**, never a claim of completeness:
+#'   FY1967 is a census year in which only 97 of Wisconsin's 608 cities
+#'   report. `n_units_reporting` is the number that tells the truth.
 #' @return Tibble with columns `year`, `layer`, `canonical_govid`, `gov_name`,
 #'   `spend_subtype`, `category`, `amt_nominal`, optional `amt_real` /
-#'   `amt_per_capita_nominal` / `amt_per_capita_real`, `codes_included`,
-#'   `aggregate_fallback`, `scope_note`, `notes`. Carries a `provenance`
-#'   attribute with `verb = "cog_geographic_rollup"` and `layers`.
+#'   `amt_per_capita_nominal` / `amt_per_capita_real`, optional `pop_source`,
+#'   `codes_included`, `aggregate_fallback`, `scope_note`, `notes`. Carries a
+#'   `provenance` attribute with `verb = "cog_geographic_rollup"`, `layers`,
+#'   and `rollup$included_govids` / `rollup$excluded_govids`.
 #' @export
 cog_geographic_rollup <- function(govids, category, years,
-                                  per_capita = FALSE, adjust_to_year = NULL) {
+                                  per_capita = FALSE, adjust_to_year = NULL,
+                                  expenditure_concept = c("primary", "direct", "total"),
+                                  coverage = c("all", "census", "consistent")) {
   call <- match.call()
+  expenditure_concept <- match.arg(expenditure_concept)
+  coverage <- .validate_coverage(coverage)
+  if (identical(expenditure_concept, "total")) {
+    .abort_concept_not_aggregatable("cog_geographic_rollup")
+  }
   .validate_rollup_layers(govids)
 
-  # Accept character vector OR a data.frame with canonical_govid per layer,
-  # so cog_gov_search() output can be piped into one of the layer slots.
   govids <- lapply(govids, .coerce_govid_input, arg = "govids[[layer]]")
   if (any(lengths(govids) == 0L)) {
     cli::cli_abort("Each layer in `govids` must be non-empty after coercion.")
@@ -41,16 +80,44 @@ cog_geographic_rollup <- function(govids, category, years,
     layer           = rep(layer_names, lengths(govids))
   )
 
-  r <- cog_spending(all_govids, years, category, per_capita, adjust_to_year)
+  # coverage = "census" drops non-census years BEFORE the query rather than
+  # after: a sample year's rows are not wanted at all, and fetching them only
+  # to discard them would also let them into the coverage table.
+  years <- .apply_census_years(years, coverage, "cog_geographic_rollup")
+
+  r <- cog_spending(all_govids, years, category, per_capita, adjust_to_year,
+                    expenditure_concept = expenditure_concept)
   r <- dplyr::left_join(r, layer_map, by = "canonical_govid",
                         relationship = "many-to-many")
   r$scope_note <- .rollup_scope_note(r$layer)
+
+  if (identical(coverage, "consistent")) {
+    r <- .filter_consistent(r, years)
+  }
+
+  excluded <- character(0)
+  if (isTRUE(per_capita) && "pop_source" %in% names(r)) {
+    drop <- r$pop_source == "unavailable"
+    excluded <- unique(r$canonical_govid[drop])
+    r <- r[!drop, , drop = FALSE]
+  }
+  included <- unique(r$canonical_govid)
+
   r <- .reorder_rollup_cols(r)
 
   prov <- attr(r, "provenance")
   prov$verb   <- "cog_geographic_rollup"
   prov$call   <- paste(deparse(call), collapse = " ")
   prov$layers <- layer_names
+  prov$rollup <- list(
+    included_govids = included,
+    excluded_govids = excluded
+  )
+  # n_units_expected is the universe the CALLER named -- the govids passed in
+  # -- not the national universe. That is what makes the ratio meaningful:
+  # "597 of the 608 Wisconsin cities you asked about reported in FY2012".
+  prov$coverage_mode <- coverage
+  prov$coverage <- .coverage_table(r, years, length(unique(all_govids)))
   attr(r, "provenance") <- prov
 
   r

R/search.R

@@ -6,8 +6,11 @@
 #' the cross-vintage canonical-government registry. Operates in two modes:
 #'
 #' * **Utility mode** (single `name`, the original behavior): returns all
-#'   rows whose `gov_name` matches the regex case-insensitively, sorted by
-#'   `population_acs` descending. Useful for exploratory lookups.
+#'   rows whose `gov_name` contains `name` as a **literal, case-insensitive
+#'   substring**, sorted by `population_acs` descending. Useful for
+#'   exploratory lookups. Regex metacharacters in `name` are escaped, so a
+#'   government is findable by its own complete name even when that name
+#'   contains parentheses or a period.
 #' * **Basket mode** (`length(name) > 1`): resolves each input row to a
 #'   single canonical govid and returns a tibble in input order, suitable
 #'   for piping straight into [cog_spending()] / [cog_revenue()] /
@@ -19,7 +22,8 @@
 #' 1. Filter `canonical_fips_xwalk` by `state` and (if non-NA) `type`.
 #' 2. **Exact pass:** case-insensitive equality against `gov_name`.
 #'    Single hit -> resolved. Multiple -> step 4.
-#' 3. **Substring fallback:** case-insensitive regex against `gov_name`.
+#' 3. **Substring fallback:** case-insensitive literal substring against
+#'    `gov_name` (metacharacters escaped).
 #'    Single hit -> resolved (`match_method = "substring"`). Zero hits ->
 #'    `status = "no_match"`. Multiple hits -> step 4.
 #' 4. **Disambiguation:** if matches share one `govs_type`, pick the
@@ -48,7 +52,7 @@
 #'   [cog_spending()], [cog_revenue()].
 #' @examples
 #' \dontrun{
-#' # Utility mode — exploratory regex lookup
+#' # Utility mode — exploratory substring lookup
 #' cog_gov_search("broward", state = "FL")
 #'
 #' # Basket mode — resolve a known cohort
@@ -98,9 +102,16 @@ cog_gov_search <- function(name = NULL, state = NULL, type = NULL) {
     if (!is.character(name) || length(name) != 1L) {
       cli::cli_abort("`name` must be a length-1 character string.")
     }
+    # Escaped, so `name` is a literal case-insensitive substring -- the same
+    # treatment basket mode has always given it. Interpolating it raw made a
+    # government unfindable by its own name whenever that name contains a
+    # metacharacter (FREDONIA (BRISCOE) CITY), turned a bare "." into a
+    # match-everything wildcard, and let malformed pattern text reach the
+    # engine as an error -- which cog-api surfaced as a 500, reachable by
+    # typing a real name one character at a time (uscogdata#16, F-025).
     preds <- c(preds,
                sprintf("regexp_matches(gov_name, %s, 'i')",
-                       .sql_lit_chr(name)))
+                       .sql_lit_chr(.escape_regex(name))))
   }
   if (!is.null(state)) {
     st_fips <- .coerce_state_to_fips(state)
@@ -126,17 +137,19 @@ cog_gov_search <- function(name = NULL, state = NULL, type = NULL) {
     canonical_govid = character(0), gov_name = character(0),
     govs_type = integer(0), type_label = character(0),
     fips_state = character(0), fips_county = character(0),
-    fips_place = character(0), first_year = integer(0),
-    last_year = integer(0), population_acs = integer(0),
-    confidence = character(0)
+    fips_place = character(0), legacy_govs_id = character(0),
+    first_year = integer(0), last_year = integer(0),
+    census_geoid = character(0), population_acs = integer(0),
+    pop_confidence = character(0), id_source = character(0)
   )
 }
 
 #' @noRd
 .escape_regex <- function(x) {
   # Backslash-escape POSIX regex metacharacters so `name` is treated as a
-  # literal substring in the DuckDB regexp_matches call (substring fallback
-  # only; utility-mode intentionally preserves regex behavior).
+  # literal substring in the DuckDB regexp_matches call. Used by BOTH modes:
+  # utility mode used to interpolate raw, which was a defect rather than a
+  # feature -- see the call site and uscogdata#16.
   gsub("([\\^$.|?*+(){}\\[\\]])", "\\\\\\1", x, perl = TRUE)
 }
 

R/series_breaks.R

@@ -0,0 +1,54 @@
+# R/series_breaks.R
+# Populates prov$series_break_refs (schema in inst/schemas/provenance-v1.json
+# defines the field; it was always present but always empty pre-Phase-R2)
+# with the ids of any catalogued series break whose fin_code appears among
+# the result's observed item codes and whose break_year falls inside the
+# requested year span -- the "break warnings in the provenance envelope"
+# spec § 5 promises downstream consumers (cog-api passes provenance through
+# verbatim). schema_version >= 5 only: series_breaks_pq isn't registered on
+# an older corpus.
+
+#' @noRd
+.build_series_break_refs <- function(con, codes_observed, years, schema_version) {
+  if (schema_version < 5L || length(codes_observed) == 0L) return(character(0))
+  sql <- sprintf(
+    "SELECT DISTINCT break_id
+     FROM series_breaks_pq
+     WHERE fin_code IN (%s) AND fin_code <> 'ALL'
+       AND break_year BETWEEN %d AND %d
+     ORDER BY break_id",
+    .sql_lit_chr(codes_observed), min(as.integer(years)), max(as.integer(years))
+  )
+  DBI::dbGetQuery(con, sql)$break_id
+}
+
+#' Corpus-wide caveats: catalogued breaks whose `fin_code` is the literal
+#' `"ALL"` rather than an item code. They qualify the whole result, so they
+#' cannot be matched the way `.build_series_break_refs()` matches -- no row's
+#' `item_code` is ever `"ALL"`, which is exactly why they reached no user
+#' before uscogdata#19. Selection is on the break_year window alone: which
+#' codes a result happens to contain is irrelevant to a caveat about the
+#' corpus.
+#'
+#' All four catalogued entries are *boundary* caveats (dollar precision
+#' across 1976/1977, imputation exclusion from 2002, the dense -> sparse
+#' representation change at 2012, the id scheme change at 2017), so the same
+#' `break_year BETWEEN min(years) AND max(years)` rule the code-specific
+#' path uses is the right one -- a request that never crosses the boundary
+#' is not affected by it.
+#'
+#' Returned separately from `series_break_refs` so a consumer can tell a
+#' whole-result caveat from a break in one series; the two are disjoint by
+#' construction.
+#' @noRd
+.build_corpus_break_refs <- function(con, years, schema_version) {
+  if (schema_version < 5L || length(years) == 0L) return(character(0))
+  sql <- sprintf(
+    "SELECT DISTINCT break_id
+     FROM series_breaks_pq
+     WHERE fin_code = 'ALL' AND break_year BETWEEN %d AND %d
+     ORDER BY break_id",
+    min(as.integer(years)), max(as.integer(years))
+  )
+  DBI::dbGetQuery(con, sql)$break_id
+}

R/session.R

@@ -5,13 +5,14 @@
 #' @noRd
 cog_open <- function(url = .resolve_url(),
                      cache_dir = .resolve_cache_dir()) {
+  .check_url_configured(url)
   if (!dir.exists(cache_dir)) dir.create(cache_dir, recursive = TRUE)
 
   con <- DBI::dbConnect(duckdb::duckdb())
   DBI::dbExecute(con, "INSTALL httpfs; LOAD httpfs;")
 
   manifest <- .fetch_or_cache_manifest(url, cache_dir)
-  .validate_schema(manifest, expected_version = 3L)
+  .validate_schema(manifest, supported = c(4L, 5L, 6L, 7L))
   .validate_scope(manifest)
 
   .register_views(con, url, manifest)
@@ -94,4 +95,7 @@ cog_close <- function() {
   }
   .uscogdata_env$con      <- NULL
   .uscogdata_env$manifest <- NULL
+  .uscogdata_env$balance_caveats_shown <- NULL
+  # Memoised corpus-constant; a different corpus may be mounted next.
+  .uscogdata_env$balance_coverage_windows <- NULL
 }

R/spending.R

@@ -1,5 +1,57 @@
 # R/spending.R
 
+# The three expenditure concepts (uscogdata#11), as sets of the crosswalk's
+# `spend_subtype` values. Classification is crosswalk membership, never
+# item-code first letters: prefix Y alone spans revenue (Y01/Y02),
+# expenditure (Y05/Y06) and balance codes, so no first-letter allowlist can
+# route it (finding F-018).
+#
+#   primary = operations + capital + assistance           (the default)
+#   direct  = primary + interest + insurance_benefits     (Census Direct Expenditure)
+#   total   = direct + intergovernmental                  (via the ig_* views)
+#
+# Census manual section 5.2.2.1: Direct Expenditure is ALL expenditure other
+# than intergovernmental -- including payments to retirees, i.e. insurance
+# trust benefits. Verified against Census's own published FY2020 state
+# aggregates (20statetypepu.txt): `total` reproduces the published
+# expenditure sum to the dollar; omitting insurance benefits understates
+# California's Direct by 10.9%.
+.spend_subtypes_primary <- c("operations", "capital", "assistance")
+.spend_subtypes_direct  <- c(.spend_subtypes_primary, "interest", "insurance_benefits")
+
+#' @noRd
+.expenditure_concept_subtypes <- function(concept) {
+  switch(concept,
+    primary = .spend_subtypes_primary,
+    # "total" = the direct subtypes here PLUS the intergovernmental leg,
+    # which travels through the ig_* views rather than this scope (see
+    # .build_verb_sql()).
+    direct  = ,
+    total   = .spend_subtypes_direct
+  )
+}
+
+# The two revenue concepts (uscogdata#12), again as crosswalk subtype sets.
+# Census's manual section 4.3 defines the first by SUBTRACTING from the second
+# -- "General revenue comprises all revenue except that classified as liquor
+# store, utility, or insurance trust revenue" -- giving the identity
+#
+#   Total Revenue = General + Utility + Liquor Store + Insurance Trust
+#
+# Verified against Census's own computed concept fields (IndFin FY2012,
+# Wisconsin state): 31,410,686 + 0 + 0 + 4,469,906 = 35,880,592, exact.
+.revenue_subtypes_general <- c("own_source", "federal", "state", "local_aid")
+.revenue_subtypes_total   <- c(.revenue_subtypes_general, "utility",
+                               "liquor_store", "insurance_trust")
+
+#' @noRd
+.revenue_concept_subtypes <- function(concept) {
+  switch(concept,
+    general = .revenue_subtypes_general,
+    total   = .revenue_subtypes_total
+  )
+}
+
 #' Summarized spending by category
 #'
 #' One row per `(year, canonical_govid, spend_subtype, category)`. Amounts are
@@ -13,75 +65,417 @@
 #' @param category Character vector of category names (from
 #'   `summary_categories.category`), or `NULL` for all categories.
 #' @param per_capita If `TRUE`, adds `amt_per_capita_nominal` (and
-#'   `amt_per_capita_real` when `adjust_to_year` is set) using
-#'   `population_acs` from the canonical xwalk.
+#'   `amt_per_capita_real` when `adjust_to_year` is set) using the per-year
+#'   Census F-33 population from `gov_population_yearly`. Result also gains
+#'   a `pop_source` column with values `"census_f33"` or `"unavailable"`
+#'   (the latter for gov types 4/5 and any row whose population is missing
+#'   in that year).
 #' @param adjust_to_year Integer base year for CPI-U real-dollar conversion,
 #'   or `NULL` for nominal only.
+#' @param basis `"harmonized"` (default) sums item codes through the
+#'   cross-vintage harmonization mapping (folding series-break-affected
+#'   codes onto a comparable target and excluding aggregate / discontinued
+#'   rows -- see the `harmonization` block in `cog_explain()`); `"raw"`
+#'   reproduces the pre-Phase-R2 behavior (published item codes, no
+#'   folding). On a corpus with `schema_version < 5` (no harmonization
+#'   tables), `basis` silently resolves to `"raw"` when left at its default
+#'   and the resolution is recorded in the provenance; explicitly passing
+#'   `basis = "harmonized"` on such a corpus aborts. Ignored when `recipe`
+#'   is set (see below).
+#' @param recipe Optional harmonization recipe id (see [cog_recipes()]) for
+#'   multi-code cross-vintage series that a 1:1 harmonized_code mapping
+#'   can't express (e.g. a wide-era aggregate that only splits into leaf
+#'   codes in the modern era). Mutually exclusive with `category`. The
+#'   result's subtype column reads `"recipe"` and `category` reads the
+#'   recipe's label. Requires `schema_version >= 5`. A recipe query bypasses
+#'   `basis` entirely (it joins `long` directly rather than going through
+#'   the `*_annotated`/`*_annotated_harmonized` views), so the `basis`
+#'   argument is ignored and the result's provenance reports
+#'   `basis = "recipe"` with an inert `harmonization` block (`applied =
+#'   FALSE`, pointing at the `recipe` block instead) rather than a
+#'   possibly-misleading `"harmonized"`/`"raw"` value.
+#' @param expenditure_concept Which spending concept to return. Concepts are
+#'   defined as sets of the crosswalk's `spend_subtype` values -- never as
+#'   item-code first letters, which cannot classify correctly (prefix `Y`
+#'   alone spans revenue, expenditure, and balance codes):
+#'
+#'   * `"primary"` (default) -- the government's own service provision:
+#'     `operations` + `capital` + `assistance` subtypes.
+#'   * `"direct"` -- Census's published Direct Expenditure: `primary` plus
+#'     `interest` (interest on debt) and `insurance_benefits` (insurance
+#'     trust benefit payments, e.g. pensions -- Census manual section
+#'     5.2.2.1 includes payments to retirees in Direct).
+#'   * `"total"` -- `direct` plus the intergovernmental leg: payments to
+#'     local governments (`M` codes), to the state government (`L` codes,
+#'     excluding the `L--` family-total rollup), and state payments to
+#'     school systems (`Q11`/`Q12`/`Q18`), so results gain rows with
+#'     `spend_subtype == "intergovernmental"`. Requires the active corpus's
+#'     `summary_categories` to carry M/L rows (added by cog_pipeline PR
+#'     #59); aborts with class `uscogdata_ig_categories_unsupported` on an
+#'     older corpus rather than silently under-reporting. Mutually
+#'     exclusive with `recipe` (a recipe already defines its own component
+#'     codes).
+#'
+#'   **Do not sum `"total"` results across levels of government** (e.g.
+#'   state + county + city): a state's `M12` payment to a school district is
+#'   the same dollar the district reports as its own direct `E12`, so
+#'   summing both double-counts it. This matters in particular with
+#'   [cog_geographic_rollup()], which sums across exactly that kind of
+#'   multi-layer government set.
+#'
+#'   In the legacy wide era (<= FY2011), some functions are published ONLY
+#'   as an aggregate-flagged family total (e.g. Corrections' `E04`/`E05`
+#'   split), which the Direct leg excludes by construction but the IG leg
+#'   deliberately keeps (see `inst/sql/24-ig_long.sql`). For a `"total"`
+#'   query, any (year, category) where this leaves intergovernmental rows
+#'   with NO Direct counterpart is flagged: the affected rows' `notes`
+#'   name the harmonization recipe that recovers the missing Direct
+#'   component (when one exists), and
+#'   `provenance$expenditure_concept_direct_suppressed` is `TRUE` -- the
+#'   figure in those rows is the intergovernmental leg alone, not Direct +
+#'   IG.
+#' @param complete If `TRUE`, fill the requested grid so that a cell the
+#'   corpus does not carry still appears, labelled with **why** it is
+#'   missing, and add a `value_source` column to every row:
+#'
+#'   * `"reported"` — the corpus carries this cell.
+#'   * `"census_zero"` — dense-source year (`<= FY2011`), cell absent:
+#'     Census published `$0`. `amt_nominal` is `0`.
+#'   * `"not_reported"` — sparse-source year (`>= FY2012`), cell absent: the
+#'     government did not report, and the value is unknown. `amt_nominal` is
+#'     `NA`, **not** `0` — writing a zero there would invent data.
+#'
+#'   The grid comes from the corpus's `code_set` table, scoped to each
+#'   government's own type, so a county is never filled with cells only a
+#'   state can report. Reported rows are passed through untouched.
+#'
+#'   Defaults to `FALSE` (the historical behaviour: absent cells simply do
+#'   not appear). Needs a corpus published from 2026-07-29 onward, which is
+#'   when `representation`/`code_set` began shipping; aborts with class
+#'   `uscogdata_representation_unavailable` otherwise. Not available with
+#'   `recipe` or with `expenditure_concept = "total"` (class
+#'   `uscogdata_complete_unsupported`) — neither draws its cells from
+#'   `code_set`.
 #' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
 #'   `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`,
 #'   optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
-#'   `codes_included`, `aggregate_fallback`, `notes`. Carries a `provenance`
-#'   attribute matching `inst/schemas/provenance-v1.json`.
+#'   optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`,
+#'   and `value_source` when `complete = TRUE`.
+#'   Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`,
+#'   whose `completion` block reports `applied`, `rows_filled`, and the
+#'   per-year `absence_means` rule that was applied.
 #' @export
 cog_spending <- function(govid, years, category = NULL,
-                         per_capita = FALSE, adjust_to_year = NULL) {
+                         per_capita = FALSE, adjust_to_year = NULL,
+                         basis = c("harmonized", "raw"), recipe = NULL,
+                         expenditure_concept = c("primary", "direct", "total"),
+                         complete = FALSE) {
+  # flow_prefixes no longer classifies rows (crosswalk subtype membership
+  # does, per expenditure_concept) -- it only scopes the recipe-suggestion
+  # machinery to this verb's recipe families (see R/suggestions.R; the
+  # catalog only has E/F/G-component direct-expenditure recipes).
   .verb_spendrev(
     verb          = "cog_spending",
-    view          = "spending_annotated",
+    view_base     = "spending_annotated",
     subtype_col   = "spend_subtype",
+    flow_prefixes = c("E", "F", "G"),
     call          = match.call(),
     govid         = govid,
     years         = years,
     category      = category,
     per_capita    = per_capita,
-    adjust_to_year = adjust_to_year
+    adjust_to_year = adjust_to_year,
+    basis         = basis,
+    recipe        = recipe,
+    expenditure_concept = expenditure_concept,
+    complete      = complete
   )
 }
 
 #' @noRd
-.verb_spendrev <- function(verb, view, subtype_col, call,
+.abort_concept_not_aggregatable <- function(verb) {
+  cli::cli_abort(c(
+    "{.code expenditure_concept = \"total\"} cannot be used in {.fn {verb}}.",
+    "*" = "Use {.code expenditure_concept = \"primary\"} (the default) or \\
+           {.code \"direct\"} for any comparison or sum that spans more than \\
+           one government.",
+    "i" = "Why: Census \"Total\" is a government's own Direct spending PLUS the \\
+           money it hands to other governments. The receiving government reports \\
+           that same dollar again as its own Direct when it actually spends it, \\
+           so combining Total across governments double-counts intergovernmental \\
+           transfers.",
+    "i" = "For one government's own Total, use \\
+           {.code cog_spending(expenditure_concept = \"total\")}."
+  ), class = "uscogdata_concept_not_aggregatable")
+}
+
+#' @noRd
+.verb_spendrev <- function(verb, view_base, subtype_col, flow_prefixes, call,
                            govid, years, category,
-                           per_capita, adjust_to_year) {
+                           per_capita, adjust_to_year,
+                           basis = c("harmonized", "raw"), recipe = NULL,
+                           expenditure_concept = c("primary", "direct", "total"),
+                           revenue_concept = c("general", "total"),
+                           complete = FALSE) {
+  basis_explicit <- length(basis) == 1L
+  basis <- match.arg(basis, c("harmonized", "raw"))
+  # match.arg() itself throws a base `simpleError`, not an rlang-classed
+  # condition; wrap it so an invalid expenditure_concept aborts consistently
+  # with the rest of this package's validation (cli::cli_abort -> rlang_error).
+  expenditure_concept <- tryCatch(
+    match.arg(expenditure_concept, c("primary", "direct", "total")),
+    error = function(e) {
+      cli::cli_abort(
+        "`expenditure_concept` must be one of {.val primary}, {.val direct}, or {.val total}.",
+        class = "uscogdata_invalid_expenditure_concept",
+        parent = e
+      )
+    }
+  )
+
+  revenue_concept <- tryCatch(
+    match.arg(revenue_concept, c("general", "total")),
+    error = function(e) {
+      cli::cli_abort(
+        "`revenue_concept` must be one of {.val general} or {.val total}.",
+        class = "uscogdata_invalid_revenue_concept",
+        parent = e
+      )
+    }
+  )
+
+  # The concept's subtype scope. Every code path below -- the verb SQL, the
+  # harmonization exclusion count, and the complete = TRUE grid -- is scoped
+  # by crosswalk subtype membership, never by item-code prefix. The
+  # expenditure "total" concept's extra intergovernmental leg is the one
+  # exception: it travels through the ig_* views rather than this scope,
+  # because its legacy rows are aggregate-flagged.
+  subtype_scope <- if (identical(subtype_col, "spend_subtype")) {
+    .expenditure_concept_subtypes(expenditure_concept)
+  } else {
+    .revenue_concept_subtypes(revenue_concept)
+  }
+
   govid <- .coerce_govid_input(govid, arg = "govid")
-  .validate_verb_inputs(govid, years, category, per_capita, adjust_to_year)
+  .validate_verb_inputs(govid, years, category, per_capita, adjust_to_year,
+                        recipe)
+
+  if (!is.null(recipe) && identical(expenditure_concept, "total")) {
+    cli::cli_abort(c(
+      "`recipe` and `expenditure_concept = \"total\"` are mutually exclusive.",
+      i = "A recipe defines its own component codes; pass one or the other.",
+      i = "For a recipe's intergovernmental counterpart, use the matching IG recipe (e.g. `corrections_ig_local_combined`)."
+    ), class = "uscogdata_recipe_concept_conflict")
+  }
+
+  # .verb_spendrev() is shared with cog_revenue(), which never exposes
+  # expenditure_concept and always resolves it to the default -- so nothing
+  # on the public API can reach this today. But it's a cheap guard against a
+  # future call (direct or via a modified cog_revenue()) that would UNION
+  # the IG leg's expenditure M/L/Q rows into a revenue result, which has no
+  # matching IG view and no sensible meaning.
+  if (identical(expenditure_concept, "total") &&
+      !identical(view_base, "spending_annotated")) {
+    cli::cli_abort(
+      paste0(
+        "`expenditure_concept = \"total\"` is only supported for spending ",
+        "(view_base = \"spending_annotated\"); got view_base = ",
+        "{.val {view_base}}."
+      ),
+      class = "uscogdata_expenditure_concept_unsupported"
+    )
+  }
+
+  complete <- isTRUE(complete)
+  if (complete && !is.null(recipe)) {
+    .abort_complete_unsupported(
+      "A recipe defines its own component codes and never goes through `summary_categories`, so there is no grid to fill from.",
+      "Query the recipe without `complete`, or use a category query with `complete = TRUE`."
+    )
+  }
+  if (complete && identical(expenditure_concept, "total")) {
+    .abort_complete_unsupported(
+      "The intergovernmental leg deliberately keeps aggregate-flagged rows (see `inst/sql/24-ig_long.sql`), so its cells are not the ones `code_set` describes.",
+      "Use `expenditure_concept = \"direct\"` with `complete = TRUE`, or drop `complete`."
+    )
+  }
 
   years   <- as.integer(years)
   if (!is.null(adjust_to_year)) adjust_to_year <- as.integer(adjust_to_year)
 
   con <- .ensure_session()
+  manifest <- .uscogdata_env$manifest
   scope <- .check_govids_in_scope(govid)
+  if (complete) .require_representation(con, manifest)
 
-  sql <- .build_verb_sql(view, subtype_col, govid, years, category)
+  resolved <- .resolve_basis(basis, basis_explicit, manifest)
+
+  recipe_block <- NULL
+  category_for_prov <- category
+  if (!is.null(recipe)) {
+    .require_schema_v5(con, manifest, "recipe =")
+    .validate_recipe_id(con, recipe)
+    comps <- .recipe_components(con, recipe)
+    recipe_label <- comps$label[[1]]
+    result <- .run_recipe(con, recipe, govid, years)
+    sql <- attr(result, "sql_query")
+    result <- .shape_recipe_result(result, subtype_col, recipe_label)
+    recipe_block <- list(
+      recipe_id = recipe, label = recipe_label,
+      components = .df_to_row_list(comps)
+    )
+    category_for_prov <- recipe_label
+  } else {
+    view <- .select_view(view_base, resolved$basis)
+    ig_view <- if (identical(expenditure_concept, "total")) {
+      .require_ig_categories(con)
+      .select_ig_view(resolved$basis)
+    } else {
+      NULL
+    }
+    sql <- .build_verb_sql(view, subtype_col, govid, years, category, ig_view,
+                           subtype_scope)
     result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
+  }
+
+  # Fill BEFORE per_capita / inflation so the added cells get the same
+  # treatment as reported ones: a census_zero stays $0 per capita and in real
+  # dollars, and a not_reported stays NA through both rather than becoming a
+  # spurious 0.
+  completion <- list(applied = FALSE, rows_filled = 0L, absence_means = list())
+  if (complete) {
+    result <- .complete_result(result, con, subtype_col, govid, years,
+                               category, subtype_scope)
+    completion <- attr(result, ".completion")
+    attr(result, ".completion") <- NULL
+  }
 
   if (per_capita) result <- .attach_per_capita(result, con, govid)
   if (!is.null(adjust_to_year)) {
     result <- .attach_real_dollars(result, adjust_to_year, per_capita)
   }
 
-  result$notes <- .notes_column(result)
+  # A recipe result doesn't go through spending_annotated(_harmonized) /
+  # revenue_annotated(_harmonized) at all -- .run_recipe()'s generic join
+  # reads `long` directly -- so `basis` and the `harmonization` exclusion
+  # count (which is itself computed from `long`, independent of which view
+  # a non-recipe query used) would describe a code path this result never
+  # took. Rather than report a technically-still-computed but misleading
+  # basis = "harmonized"/"raw" + harmonization$applied combo, recipe
+  # results report basis = "recipe" and an explicit, inert harmonization
+  # block pointing at the `recipe` block instead. Task 12 (cog-api) passes
+  # provenance through verbatim, so this needs to be unambiguous rather
+  # than technically-defensible-but-confusing.
+  if (!is.null(recipe)) {
+    basis_for_prov <- "recipe"
+    basis_note_for_prov <- NA_character_
+    harmonization <- list(
+      applied = FALSE, na_rows_excluded = 0L, na_amount_excluded = 0,
+      note = "basis/harmonization not applicable to recipe results; see the recipe block instead"
+    )
+    suggestions <- list()
+  } else {
+    basis_for_prov <- resolved$basis
+    basis_note_for_prov <- resolved$note
+    harmonization <- .build_harmonization_block(
+      con, govid, years, resolved, subtype_col, subtype_scope
+    )
+    # C1(a): gap detection must run against the Direct leg alone. `result`
+    # can also carry UNION'd intergovernmental rows (expenditure_concept =
+    # "total"), and the wide era (<= FY2011) routinely has legacy IG dollars
+    # surviving (ig_long deliberately keeps aggregate rows) for a
+    # (year, category) whose legacy Direct dollars were suppressed (spending_
+    # long/spending_long_harmonized both filter NOT is_aggregate). Passing
+    # the UNION'd result here would let a surviving IG row count as coverage
+    # and silently cancel the recipe-hint suggestion that should fire.
+    direct_leg_result <- if (identical(expenditure_concept, "total")) {
+      result[!(result[[subtype_col]] %in% "intergovernmental"), , drop = FALSE]
+    } else {
+      result
+    }
+    suggestions <- .build_suggestions(con, govid, years, category,
+                                       direct_leg_result,
+                                       resolved$basis, flow_prefixes,
+                                       .select_long_view(view_base, resolved$basis))
+  }
+
+  # C1(b): when expenditure_concept = "total", flag any row where the IG
+  # leg has dollars but the Direct leg has none for that same (year,
+  # canonical_govid, category) AND a harmonization recipe actually recovers
+  # the missing Direct dollars for that exact triple -- see
+  # .detect_direct_suppressed() for why bare Direct-row absence alone is NOT
+  # sufficient (the dominant real cause is a government that simply has no
+  # direct spending in that category, which is correct, ordinary data). When
+  # a covering recipe is found, both the row-level notes and the provenance
+  # say so rather than pass silently as a plausible Total.
+  direct_suppressed_info <- if (identical(expenditure_concept, "total")) {
+    .detect_direct_suppressed(con, result, subtype_col)
+  } else {
+    list(flag = rep(FALSE, nrow(result)), notes = rep(NA_character_, nrow(result)))
+  }
+  direct_suppressed <- direct_suppressed_info$flag
+  direct_suppressed_flag <- isTRUE(any(direct_suppressed))
+
+  result$notes <- .notes_column(result, direct_suppressed_info$notes)
+
+  # Determine expenditure_concept_note: only non-empty for "total", explains
+  # how the IG leg was assembled from legacy-era aggregates. When the Direct
+  # leg is suppressed for at least one requested (year, category), append an
+  # explicit warning rather than let the base note's "Total = Direct + IG"
+  # framing stand unqualified for rows where that arithmetic didn't happen.
+  expenditure_concept_note_for_prov <- if (identical(expenditure_concept, "total")) {
+    base_note <- "Total = Direct + intergovernmental (M to local govts + L to state govts). Legacy-era IG is assembled from aggregate-flagged rows, which are year-disjoint from their modern leaf components; the L-- family total is excluded."
+    if (direct_suppressed_flag) {
+      paste0(
+        base_note,
+        " NOTE: for at least one requested (year, category) the Direct leg ",
+        "has NO rows in this corpus (a legacy aggregate-only family) -- the ",
+        "affected result rows report the intergovernmental leg alone, not ",
+        "Direct + IG. See `expenditure_concept_direct_suppressed` and each ",
+        "affected row's `notes`."
+      )
+    } else {
+      base_note
+    }
+  } else {
+    NA_character_
+  }
 
   prov <- .build_provenance(
     verb           = verb,
     call           = call,
     govid          = govid,
     years          = years,
-    category       = category,
+    category       = category_for_prov,
     per_capita     = per_capita,
     adjust_to_year = adjust_to_year,
     result         = result,
     sql            = sql,
-    subtype_col    = subtype_col
+    subtype_col    = subtype_col,
+    basis          = basis_for_prov,
+    basis_note     = basis_note_for_prov,
+    expenditure_concept = expenditure_concept,
+    expenditure_concept_note = expenditure_concept_note_for_prov,
+    expenditure_concept_direct_suppressed = direct_suppressed_flag,
+    revenue_concept = revenue_concept,
+    harmonization  = harmonization,
+    recipe         = recipe_block,
+    suggestions    = suggestions,
+    completion     = completion
   )
   prov$scope$govids_found   <- scope$found
   prov$scope$govids_missing <- scope$missing
   attr(result, "provenance") <- prov
+  attr(result, ".popyear_range") <- NULL
+
+  if (length(suggestions) > 0L) .inform_suggestions(suggestions)
+
   result
 }
 
 #' @noRd
 .validate_verb_inputs <- function(govid, years, category,
-                                  per_capita, adjust_to_year) {
+                                  per_capita, adjust_to_year, recipe = NULL) {
   if (!is.character(govid) || length(govid) == 0L) {
     cli::cli_abort("`govid` must be a non-empty character vector.")
   }
@@ -100,6 +494,71 @@ cog_spending <- function(govid, years, category = NULL,
       cli::cli_abort("`adjust_to_year` must be NULL or a length-1 integer.")
     }
   }
+  if (!is.null(recipe)) {
+    if (!is.character(recipe) || length(recipe) != 1L) {
+      cli::cli_abort("`recipe` must be NULL or a length-1 character string.")
+    }
+    if (!is.null(category)) {
+      cli::cli_abort(c(
+        "`recipe` and `category` are mutually exclusive.",
+        i = "Pass one or the other, not both."
+      ), class = "uscogdata_recipe_category_conflict")
+    }
+  }
+  invisible(TRUE)
+}
+
+#' @noRd
+.select_view <- function(view_base, basis) {
+  if (identical(basis, "harmonized")) paste0(view_base, "_harmonized") else view_base
+}
+
+#' The `*_long`/`*_long_harmonized` view behind an annotated view base --
+#' `"spending_annotated"` -> `"spending_long_harmonized"`. `.build_suggestions()`
+#' anti-joins the LONG view rather than the annotated one: they have identical
+#' row membership (the annotated views are the long views plus LEFT JOINs, see
+#' inst/sql/42-spending_annotated_harmonized.sql), but the long view is the
+#' one that actually owns the `NOT is_aggregate` + crosswalk-membership rule
+#' the suppression test is asking about.
+#' @noRd
+.select_long_view <- function(view_base, basis) {
+  .select_view(sub("_annotated$", "_long", view_base), basis)
+}
+
+#' @noRd
+.select_ig_view <- function(basis) {
+  if (identical(basis, "harmonized")) "ig_annotated_harmonized" else "ig_annotated"
+}
+
+#' Abort unless the active corpus's `summary_categories` actually carries
+#' intergovernmental (M/L) rows.
+#'
+#' The 66 M/L category rows arrived via cog_pipeline PR #59 with NO
+#' `schema_version` bump (`DESCRIPTION` still declares `MinCorpusSchema: 4`),
+#' so `schema_version` alone cannot gate `expenditure_concept = "total"` --
+#' a pre-#59 corpus can validly report schema_version 4, 5, or 6 and still
+#' have zero M/L rows in `summary_categories`. Against such a corpus,
+#' `ig_annotated`'s LEFT JOIN to `summary_categories` silently produces NA
+#' `category`/`spend_subtype` for every IG row: with a `category` filter
+#' this returns 0 rows (reads as "no intergovernmental spending" rather than
+#' "can't tell"), and with `category = NULL` every IG dollar collapses into
+#' one NA-subtype group that is invisible to the `spend_subtype ==
+#' "intergovernmental"` filter this package's own tests, roxygen, and
+#' vignette all rely on. Checking the data directly (rather than
+#' schema_version) is the only reliable gate.
+#' @noRd
+.require_ig_categories <- function(con, what = "expenditure_concept = \"total\"") {
+  n <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS n FROM summary_categories WHERE LEFT(item_code, 1) IN ('M', 'L')"
+  )$n
+  if (identical(as.integer(n), 0L)) {
+    cli::cli_abort(c(
+      sprintf("%s requires a corpus with intergovernmental category rows.", what),
+      x = "The active corpus's `summary_categories` has no M/L (intergovernmental) rows.",
+      i = "This corpus predates the intergovernmental category rows added by cog_pipeline PR #59.",
+      i = "Point USCOGDATA_URL at a newer corpus that includes the M/L summary_categories rows."
+    ), class = "uscogdata_ig_categories_unsupported")
+  }
   invisible(TRUE)
 }
 
@@ -110,7 +569,8 @@ cog_spending <- function(govid, years, category = NULL,
 }
 
 #' @noRd
-.build_verb_sql <- function(view, subtype_col, govid, years, category) {
+.build_verb_sql <- function(view, subtype_col, govid, years, category,
+                            ig_view = NULL, subtype_scope = NULL) {
   govid_lit <- .sql_lit_chr(govid)
   years_lit <- paste(as.integer(years), collapse = ",")
   category_pred <- if (is.null(category)) {
@@ -119,6 +579,39 @@ cog_spending <- function(govid, years, category = NULL,
     sprintf("AND category IN (%s)", .sql_lit_chr(category))
   }
 
+  # The concept's subtype allowlist (see .expenditure_concept_subtypes()).
+  # The base views carry every subtype of their flow (spending_annotated has
+  # all five non-IG expenditure subtypes); the concept narrows here. For
+  # "total", the IG leg's rows are 'intergovernmental', so that value joins
+  # the allowlist exactly when ig_view is present.
+  subtype_pred <- if (is.null(subtype_scope)) {
+    ""
+  } else {
+    scope <- if (is.null(ig_view)) subtype_scope else c(subtype_scope, "intergovernmental")
+    sprintf("AND %s IN (%s)", subtype_col, .sql_lit_chr(scope))
+  }
+
+  # expenditure_concept = "total" adds the intergovernmental leg. UNION ALL,
+  # never UNION: the two legs are disjoint by crosswalk subtype (the direct
+  # view excludes 'intergovernmental'; the IG view is only that), so
+  # de-duplication would be pure cost, and a silent row-drop if two
+  # governments ever reported identical values.
+  source_expr <- if (is.null(ig_view)) {
+    view
+  } else {
+    sprintf("(SELECT * FROM %s UNION ALL SELECT * FROM %s)", view, ig_view)
+  }
+
+  # bool_or(), not bool_and(): a no-op for the Direct/revenue legs (those
+  # views filter NOT is_aggregate, so no row in any group is ever aggregate),
+  # but load-bearing for the IG leg, which deliberately keeps aggregate rows
+  # (see inst/sql/24-ig_long.sql). The wide era is dense -- every government
+  # has a row for every code in a family, most of them $0 -- so a $0 leaf
+  # commonly lands in the same (year, gov, subtype, category) group as the
+  # real aggregate row. bool_and() would then read FALSE for that group even
+  # though its dollars came entirely from an aggregate row, silently
+  # suppressing the "Aggregate fallback applied" note on exactly the rows
+  # this feature exists to surface.
   sprintf(
     "SELECT
        year,
@@ -128,14 +621,15 @@ cog_spending <- function(govid, years, category = NULL,
        category,
        SUM(amt) * 1000.0 AS amt_nominal,
        string_agg(DISTINCT item_code, ',' ORDER BY item_code) AS codes_included,
-       bool_and(is_aggregate) AS aggregate_fallback
+       bool_or(is_aggregate) AS aggregate_fallback
      FROM %2$s
      WHERE canonical_govid IN (%3$s)
        AND year IN (%4$s)
        %5$s
+       %6$s
      GROUP BY year, canonical_govid, gov_name, xwalk_gov_name, %1$s, category
      ORDER BY year, canonical_govid, %1$s, category",
-    subtype_col, view, govid_lit, years_lit, category_pred
+    subtype_col, source_expr, govid_lit, years_lit, category_pred, subtype_pred
   )
 }
 
@@ -143,18 +637,32 @@ cog_spending <- function(govid, years, category = NULL,
 .attach_per_capita <- function(result, con, govid) {
   if (nrow(result) == 0L) {
     result$amt_per_capita_nominal <- numeric(0)
+    result$pop_source <- character(0)
+    attr(result, ".popyear_range") <- integer(0)
     return(result)
   }
+  years_lit <- paste(unique(as.integer(result$year)), collapse = ",")
   sql <- sprintf(
-    "SELECT canonical_govid, population_acs
-     FROM canonical_fips_xwalk
-     WHERE canonical_govid IN (%s)",
-    .sql_lit_chr(govid)
+    "SELECT canonical_govid, year, population, popyear
+     FROM gov_population_yearly
+     WHERE canonical_govid IN (%s)
+       AND year IN (%s)",
+    .sql_lit_chr(govid), years_lit
   )
   pops <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
-  result <- dplyr::left_join(result, pops, by = "canonical_govid")
-  result$amt_per_capita_nominal <- result$amt_nominal / result$population_acs
-  result$population_acs <- NULL
+  result <- dplyr::left_join(result, pops,
+                             by = c("canonical_govid", "year"))
+  result$amt_per_capita_nominal <- result$amt_nominal / result$population
+  result$pop_source <- ifelse(is.na(result$population),
+                              "unavailable", "census_f33")
+  py <- result$popyear[!is.na(result$popyear)]
+  attr(result, ".popyear_range") <- if (length(py) > 0L) {
+    as.integer(c(min(py), max(py)))
+  } else {
+    integer(0)
+  }
+  result$population <- NULL
+  result$popyear <- NULL
   result
 }
 
@@ -174,12 +682,157 @@ cog_spending <- function(govid, years, category = NULL,
   result
 }
 
+#' Detect rows where expenditure_concept = "total" is reporting the
+#' intergovernmental leg with NO Direct counterpart in the same (year,
+#' canonical_govid, category) group AND a harmonization recipe actually
+#' recovers the missing Direct dollars for that exact (year, canonical_govid,
+#' category) triple.
+#'
+#' Bare Direct-row absence is deliberately NOT sufficient on its own: the
+#' dominant real cause of "no Direct sibling row" is a government that simply
+#' has no direct spending in that category (e.g. a state that funds K-12
+#' entirely through school districts), which is correct, ordinary data, not
+#' suppression. Genuine suppression -- a legacy aggregate-only family whose
+#' Direct-leg basis query excludes it by construction (spending_long/
+#' spending_long_harmonized both filter NOT is_aggregate) -- always has a
+#' covering harmonization recipe, because that is exactly what the recipe
+#' catalog exists to recover (see R/suggestions.R and `cog_recipes()`). So
+#' checking "does a recipe actually cover this triple" cleanly separates the
+#' two cases instead of conflating them.
+#'
+#' Returns `list(flag, notes)`, both the same length as `result`: `flag` is
+#' `TRUE` only for the `spend_subtype == "intergovernmental"` row(s) in a
+#' suppressed group, and `notes` names the recovering recipe(s) for those
+#' rows (`NA` everywhere else).
 #' @noRd
-.notes_column <- function(result) {
-  if (nrow(result) == 0L) return(character(0))
-  ifelse(
-    isTRUE(result$aggregate_fallback) | result$aggregate_fallback %in% TRUE,
-    "Aggregate fallback applied; see cog_explain()",
-    ""
+.detect_direct_suppressed <- function(con, result, subtype_col) {
+  n <- nrow(result)
+  empty_notes <- rep(NA_character_, n)
+  if (n == 0L) return(list(flag = logical(0), notes = character(0)))
+  is_ig <- result[[subtype_col]] %in% "intergovernmental"
+  if (!any(is_ig)) return(list(flag = rep(FALSE, n), notes = empty_notes))
+
+  key <- paste(result$year, result$canonical_govid, result$category, sep = "\r")
+  has_direct <- key %in% unique(key[!is_ig])
+  candidate <- is_ig & !has_direct
+
+  flag <- rep(FALSE, n)
+  notes <- empty_notes
+  if (!any(candidate)) return(list(flag = flag, notes = notes))
+
+  idx <- which(candidate)
+  rows <- unique(result[idx, c("year", "canonical_govid", "category")])
+  covering <- .covering_recipes(con, rows)
+  cov_key <- paste(covering$year, covering$canonical_govid, covering$category,
+                    sep = "\r")
+
+  for (i in idx) {
+    k <- paste(result$year[i], result$canonical_govid[i], result$category[i],
+               sep = "\r")
+    m <- match(k, cov_key)
+    if (is.na(m)) next
+    ids <- covering$recipe_ids[[m]]
+    if (length(ids) == 0L) next
+    flag[i] <- TRUE
+    notes[i] <- sprintf(
+      "Direct component is unavailable through this basis for this year; recover it via recipe = '%s' (see cog_recipes()).",
+      paste(sort(unique(ids)), collapse = "', '")
     )
   }
+  list(flag = flag, notes = notes)
+}
+
+#' For each (year, canonical_govid, category) triple potentially affected by
+#' a suppressed Direct leg, find the harmonization recipe(s) that (a) cover
+#' this `category` (share a component item_code via `summary_categories`,
+#' excluding any recipe that is itself entirely intergovernmental M/L -- the
+#' same exclusion `.build_suggestions()` applies, see I2) and (b) actually
+#' produce a `long` row for this exact (canonical_govid, year) via the same
+#' generic join `.run_recipe()` uses (component year_min/year_max +
+#' gov_type_scope, no is_aggregate filter -- a recipe's whole point is to
+#' recover data that's aggregate-only). Adds a list-column `recipe_ids`
+#' (possibly length-0) to `rows`.
+#' @noRd
+.covering_recipes <- function(con, rows) {
+  rows$recipe_ids <- vector("list", nrow(rows))
+  cats <- unique(rows$category[!is.na(rows$category)])
+  if (length(cats) == 0L) return(rows)
+
+  cand <- DBI::dbGetQuery(con, sprintf(
+    "SELECT DISTINCT sc.category, r.recipe_id
+     FROM harmonization_recipes r
+     JOIN summary_categories sc ON sc.item_code = r.component_code
+     WHERE sc.category IN (%s)
+       AND r.recipe_id NOT IN (
+         SELECT DISTINCT recipe_id FROM harmonization_recipes
+         WHERE LEFT(component_code, 1) IN ('M', 'L')
+       )",
+    .sql_lit_chr(cats)
+  ))
+  if (nrow(cand) == 0L) return(rows)
+
+  recipe_ids_all <- unique(cand$recipe_id)
+  govids <- unique(rows$canonical_govid)
+  years  <- unique(rows$year)
+  covered <- DBI::dbGetQuery(con, sprintf(
+    "SELECT DISTINCT r.recipe_id, l.canonical_govid, l.year
+     FROM long l
+     JOIN harmonization_recipes r
+       ON l.item_code = r.component_code
+      AND l.year BETWEEN r.year_min AND r.year_max
+      AND (r.gov_type_scope = 'all'
+           OR (r.gov_type_scope = 'state' AND l.type = 0)
+           OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
+     WHERE r.recipe_id IN (%s)
+       AND l.canonical_govid IN (%s)
+       AND l.year IN (%s)",
+    .sql_lit_chr(recipe_ids_all), .sql_lit_chr(govids), paste(years, collapse = ",")
+  ))
+
+  for (i in seq_len(nrow(rows))) {
+    cat_i <- rows$category[i]
+    if (is.na(cat_i)) next
+    cat_recipe_ids <- cand$recipe_id[cand$category == cat_i]
+    if (length(cat_recipe_ids) == 0L) next
+    sub <- covered[covered$canonical_govid == rows$canonical_govid[i] &
+                   covered$year == rows$year[i] &
+                   covered$recipe_id %in% cat_recipe_ids, ]
+    rows$recipe_ids[[i]] <- sort(unique(sub$recipe_id))
+  }
+  rows
+}
+
+#' @noRd
+.notes_column <- function(result, direct_suppressed_notes = NULL) {
+  n <- nrow(result)
+  if (n == 0L) return(character(0))
+  parts <- vector("list", 3L)
+  agg <- result[["aggregate_fallback"]]
+  parts[[1]] <- if (!is.null(agg)) {
+    ifelse(agg %in% TRUE,
+           "Aggregate fallback applied; see cog_explain()",
+           NA_character_)
+  } else {
+    rep(NA_character_, n)
+  }
+  ps <- result[["pop_source"]]
+  parts[[2]] <- if (!is.null(ps)) {
+    ifelse(ps == "unavailable",
+           "No population denominator available for this gov type",
+           NA_character_)
+  } else {
+    rep(NA_character_, n)
+  }
+  parts[[3]] <- if (!is.null(direct_suppressed_notes)) {
+    direct_suppressed_notes
+  } else {
+    rep(NA_character_, n)
+  }
+  out <- character(n)
+  for (i in seq_len(n)) {
+    pieces <- vapply(parts, `[[`, character(1), i)
+    pieces <- pieces[!is.na(pieces)]
+    out[i] <- if (length(pieces) == 0L) "" else paste(pieces, collapse = "; ")
+  }
+  out
+}

R/suggestions.R

@@ -0,0 +1,317 @@
+# R/suggestions.R
+# Recipe-component-driven signposting. When a basis = "harmonized" query for
+# a category comes back incomplete in some requested year -- and a
+# harmonization recipe would actually fill it for this government -- surface
+# that recipe as a suggestion. "Incomplete" has two forms, and a recipe
+# qualifies on either:
+#   1. empty_year          -- the result has no rows at all in that year.
+#   2. suppressed_component -- the result HAS rows, but a component code
+#      carries dollars the verb's own long view structurally excludes
+#      (aggregate-published, or absent from summary_categories). This is
+#      uscogdata#9: Public Welfare kept returning E74/E79 rows while dropping
+#      aggregate-only E67/E68, so form 1 never fired and the caller got a
+#      number a third too low with no signpost at all.
+#
+# This is deliberately keyed off the recipe catalog's component codes, not
+# off harmonization_map rows: no live map row carries a non-blank
+# suggested_recipe_id (the corpus's wide era exposes split families like
+# corrections functions 04+05 ONLY as aggregate rows, which basis =
+# "harmonized" excludes by construction -- there's no NA ruling to hang a
+# suggestion off of, just a leaf-code absence a recipe happens to fill).
+# See docs/phase_r_harmonization_review.md § 0.3.
+#
+# Scope is deliberately narrow: signposting only runs when the caller
+# supplied a `category` (an un-scoped, all-categories query has no single
+# coverage question to answer) and only flags a recipe when the ACTUAL
+# result has zero rows in a requested year AND the candidate recipe's own
+# generic join (same join .run_recipe() uses, including its wide-era
+# aggregate rows) produces at least one row for this government in that
+# year. Checking presence per-government (not corpus-wide) avoids false
+# positives from ordinary reporting variance -- most governments don't use
+# every sibling code in a multi-code category every year, and that is not
+# a format-boundary gap worth signposting.
+#
+# C1(a): for expenditure_concept = "total" callers, `result` here must
+# already be the Direct-leg subset (the caller filters out
+# spend_subtype == "intergovernmental" rows before calling in). A gap year
+# is "the requested year has no Direct rows", never "no rows at all" --
+# an IG row surviving on a legacy aggregate that Direct excludes must not
+# read as coverage and cancel the very suggestion that would recover it.
+
+#' Build the `prov$suggestions` list for a (non-recipe) basis = "harmonized"
+#' verb call: recipes whose generic join would fill a real gap in `result`.
+#'
+#' @param con Active DuckDB connection.
+#' @param govid Character vector of canonical_govid values (the verb's raw
+#'   `govid`).
+#' @param years Integer vector of requested years.
+#' @param category `category` argument as passed to the verb (character
+#'   vector or `NULL`; suggestions are only computed when non-NULL).
+#' @param result The verb's already-computed result tibble (post basis
+#'   query, pre per_capita/adjust_to_year), pre-filtered to the Direct leg
+#'   only when the caller's `expenditure_concept = "total"` (see C1(a)).
+#' @param basis The *resolved* basis (`"harmonized"` or `"raw"`).
+#' @param flow_prefixes The calling verb's own flow-type prefixes (e.g.
+#'   `c("E", "F", "G")` for `cog_spending()`, `c("T", "A", "U", "B", "C",
+#'   "D")` for `cog_revenue()` -- see `.verb_spendrev()`). Passed through to
+#'   `.attach_ig_counterparts()` to keep the intergovernmental-counterpart
+#'   lookup scoped to the calling verb's own flow family.
+#' @param long_view Name of the verb's own long view (from
+#'   `.select_long_view()`), passed through to `.suppressed_components()` to
+#'   measure the second qualifying path (uscogdata#9).
+#' @return List of `list(recipe_id, label, available_years, hint,
+#'   ig_recipe_id, trigger, suppressed_amount, suppressed_years,
+#'   suppressed_codes)`, possibly empty.
+#' @noRd
+.build_suggestions <- function(con, govid, years, category, result, basis,
+                                flow_prefixes, long_view) {
+  if (!identical(basis, "harmonized") || is.null(category)) return(list())
+
+  # Exclude any recipe that is ITSELF an intergovernmental (M/L) recipe --
+  # i.e. every one of its own component codes is M/L-prefixed. Without this,
+  # a category whose summary_categories rows span both a Direct family
+  # (e.g. E04/E05, "Corrections") and its M/L counterpart (M04/M05, same
+  # category since Task 1) makes the M/L recipe itself (e.g.
+  # `corrections_ig_local_combined`) a raw top-level candidate for a plain
+  # (Direct) cog_spending() call -- following that hint would silently
+  # return intergovernmental dollars under `expenditure_concept = "direct"`
+  # provenance. This is a stronger, unconditional exclusion than the
+  # flow-prefix gate below/in `.attach_ig_counterparts()`: an M/L recipe
+  # should never be suggested as a coverage-gap filler for EITHER verb, not
+  # just kept from being named as the *counterpart* of another suggestion.
+  candidates <- DBI::dbGetQuery(con, sprintf(
+    "SELECT DISTINCT recipe_id FROM harmonization_recipes
+     WHERE component_code IN (
+       SELECT DISTINCT item_code FROM summary_categories WHERE category IN (%s)
+     )
+     AND recipe_id NOT IN (
+       SELECT DISTINCT recipe_id FROM harmonization_recipes
+       WHERE LEFT(component_code, 1) IN ('M', 'L')
+     )",
+    .sql_lit_chr(category)
+  ))$recipe_id
+  if (length(candidates) == 0L) return(list())
+
+  result_years <- if (is.null(result) || nrow(result) == 0L) {
+    integer(0)
+  } else {
+    unique(as.integer(result$year))
+  }
+  gap_years <- setdiff(as.integer(years), result_years)
+
+  # Path 2 (uscogdata#9): component dollars this government holds that the
+  # verb's own view structurally excludes. Measured across ALL requested
+  # years, not just gap years -- the whole point is that a year with rows can
+  # still be missing dollars. Scoped to the calling verb's own flow_prefixes
+  # (I1) -- see `.suppressed_components()`'s own roxygen for why.
+  #
+  # This runs unconditionally whenever there are candidates -- an earlier
+  # revision of this fix wave tried a free, in-memory pre-check
+  # (`.needs_suppression_query()`) to skip the round trip on an already-
+  # covered path, but a scoped re-review measured it against the fixture and
+  # found it didn't pay for itself (it skipped ~3% of healthy calls, ~0% of
+  # the multi-govid batch shape it was meant to help, at a net cost increase
+  # once its own always-run metadata query was counted) while adding an
+  # untested exactness invariant -- that `result$codes_included` and this
+  # anti-join share the harmonized `item_code` space -- whose silent
+  # violation would kill signposting, the exact failure class uscogdata#9
+  # exists to prevent. Owner's call: keep this simple; a batch-aware
+  # optimization, if one is worth building, is a separate issue.
+  supp <- .suppressed_components(con, candidates, govid, years, long_view, flow_prefixes)
+
+  if (length(gap_years) == 0L && nrow(supp) == 0L) return(list())
+
+  meta <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf(
+    "SELECT recipe_id, any_value(label) AS label,
+            MIN(year_min) AS year_min, MAX(year_max) AS year_max
+     FROM harmonization_recipes
+     WHERE recipe_id IN (%s)
+     GROUP BY recipe_id",
+    .sql_lit_chr(candidates)
+  )))
+
+  # Path 1 (unchanged): (recipe, year) pairs the recipe's own generic join
+  # covers for this government, restricted to the gap years.
+  covered <- if (length(gap_years) == 0L) {
+    data.frame(recipe_id = character(0), year = integer(0))
+  } else {
+    DBI::dbGetQuery(con, sprintf(
+      "SELECT DISTINCT r.recipe_id, l.year
+       FROM long l
+       JOIN harmonization_recipes r
+         ON l.item_code = r.component_code
+        AND l.year BETWEEN r.year_min AND r.year_max
+        AND (r.gov_type_scope = 'all'
+             OR (r.gov_type_scope = 'state' AND l.type = 0)
+             OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
+       WHERE r.recipe_id IN (%s)
+         AND l.canonical_govid IN (%s)
+         AND l.year IN (%s)",
+      .sql_lit_chr(candidates), .sql_lit_chr(govid),
+      paste(gap_years, collapse = ",")
+    ))
+  }
+
+  suggestions <- list()
+  for (rid in candidates) {
+    empty_hit <- rid %in% covered$recipe_id
+    s_rows <- supp[supp$recipe_id == rid, , drop = FALSE]
+    supp_hit <- nrow(s_rows) > 0L
+    if (!empty_hit && !supp_hit) next
+    m <- meta[meta$recipe_id == rid, ]
+    suggestions[[length(suggestions) + 1L]] <- list(
+      recipe_id = rid,
+      label = m$label[[1]],
+      available_years = c(as.integer(m$year_min), as.integer(m$year_max)),
+      hint = sprintf("re-run with recipe = '%s'", rid),
+      # An empty year is the stronger claim -- the category returned nothing
+      # at all -- so it wins when both paths qualify. The suppressed_* fields
+      # are still populated, so an empty_year fire also reports its dollars.
+      trigger = if (empty_hit) "empty_year" else "suppressed_component",
+      suppressed_amount = if (supp_hit) sum(s_rows$suppressed_amount) else 0,
+      suppressed_years = if (supp_hit) {
+        sort(unique(as.integer(s_rows$year)))
+      } else {
+        integer(0)
+      },
+      suppressed_codes = if (supp_hit) {
+        sort(unique(unlist(strsplit(s_rows$suppressed_codes, ",", fixed = TRUE))))
+      } else {
+        character(0)
+      }
+    )
+  }
+  .attach_ig_counterparts(con, suggestions, flow_prefixes)
+}
+
+#' Attach `ig_recipe_id` to each suggestion: the intergovernmental-expenditure
+#' recipe (an M-to-local or L-to-state recipe) whose component codes cover
+#' exactly the same set of function suffixes as the firing recipe's own
+#' components, e.g. `corrections_combined`'s {E04, E05} -> suffixes {"04",
+#' "05"} matches `corrections_ig_local_combined`'s {M04, M05} -> the same
+#' {"04", "05"}. `NULL` when no such recipe exists, which also covers the
+#' case where the firing recipe already IS the IG recipe (self-matches are
+#' excluded, so an IG recipe never names itself as its own counterpart).
+#'
+#' Matching is deliberately an exact set match, not "any suffix in common":
+#' the two-digit suffix only means the same "function" across recipes that
+#' share the underlying Census functional-classification scheme (E/F/G/L/M
+#' all use "04"/"05" for corrections). M/L "combined other" codes (47/89/
+#' 91-94) reuse digits for an unrelated catch-all construct, so e.g.
+#' `general_gov_e89_wide`'s {E85, E89} -> {"85", "89"} must NOT match
+#' `ige_local_m89_wide`'s {"89", "91", "92", "93"} on the shared "89" alone.
+#' Checked by hand against the full harmonization_recipes catalog: only the
+#' corrections family (E/F/G/M, suffixes 04/05) has an exact-set match in
+#' this corpus.
+#'
+#' Exact-set suffix matching is NOT enough on its own, though: the same
+#' reused-digit problem exists ACROSS the revenue-side IG families too.
+#' `ig_local_d47_wide` (D47/D94, suffixes {"47","94"}) is an exact-set match
+#' for `ige_local_m47_wide` (M47/M94, same suffixes) even though one is
+#' intergovernmental REVENUE received from local governments and the other is
+#' intergovernmental EXPENDITURE paid to local governments -- unrelated flows
+#' that happen to reuse "47"/"94" for their own "transit/utilities" and
+#' "other/combined" catch-alls. `ig_federal_b47_wide`, `ig_state_c47_wide`,
+#' and their `*_89` siblings all collide the same way. None of this is
+#' reachable via `cog_revenue()` in the bundled fixture today (its B/C/D
+#' recipes never happen to have a covered gap year for any fixture govid),
+#' but it IS reachable via a mis-scoped `cog_spending()` call on a
+#' revenue-only category, e.g. `cog_spending(gov, category = "IG Federal")`
+#' fires `ig_federal_b47_wide`/`ig_federal_b89_wide` for real in the fixture
+#' -- so this is a live, not merely theoretical, gap.
+#'
+#' Two flow-family checks close this, both required (see
+#' `tests/testthat/test-expenditure-concept.R`, "revenue-flavored ... never
+#' receives an M/L counterpart" tests, for the pairwise verification):
+#'   1. `own_prefix %in% flow_prefixes`: the firing recipe's own component
+#'      codes must belong to the calling verb's own flow family (the same
+#'      `flow_prefixes` `.build_harmonization_block()` uses, see
+#'      `R/basis.R`). This blocks a recipe surfaced through a mis-scoped
+#'      category from ever reaching the M/L search, e.g. `cog_spending()`'s
+#'      flow_prefixes are `c("E","F","G")`, which `ig_federal_b47_wide`'s own
+#'      `"B"` is not part of.
+#'   2. `own_prefix %in% c("E","F","G")`: M/L only ever pairs with the
+#'      DIRECT-expenditure family, never with revenue (`cog_revenue()`'s
+#'      flow_prefixes already fold B/C/D in as ordinary revenue -- there is
+#'      no separate "Total" bolt-on for revenue the way `expenditure_concept`
+#'      adds one for spending) and never with ANOTHER M/L recipe (without
+#'      this check, `ige_local_m47_wide` would wrongly match sibling
+#'      `ige_state_l47_wide` on their shared {"47","94"} suffix set).
+#'      Condition 1 alone does not catch this: under `cog_revenue()`,
+#'      `ig_federal_b47_wide`'s own `"B"` IS inside revenue's own
+#'      `flow_prefixes`, so only this second, family-specific check blocks
+#'      the search.
+#' @noRd
+.attach_ig_counterparts <- function(con, suggestions, flow_prefixes) {
+  if (length(suggestions) == 0L) return(suggestions)
+
+  comp <- DBI::dbGetQuery(con,
+    "SELECT recipe_id, component_code FROM harmonization_recipes")
+  comp$prefix <- substr(comp$component_code, 1L, 1L)
+  comp$suffix <- substr(comp$component_code, 2L, nchar(comp$component_code))
+  suffix_sets <- lapply(split(comp$suffix, comp$recipe_id), function(x) sort(unique(x)))
+  prefix_sets <- lapply(split(comp$prefix, comp$recipe_id), function(x) sort(unique(x)))
+
+  ig_recipe_ids <- unique(comp$recipe_id[comp$prefix %in% c("M", "L")])
+
+  find_counterpart <- function(rid) {
+    own_prefix <- prefix_sets[[rid]]
+    own_suffix <- suffix_sets[[rid]]
+    if (is.null(own_prefix) || is.null(own_suffix)) return(NULL)
+    if (!all(own_prefix %in% flow_prefixes)) return(NULL)
+    if (!all(own_prefix %in% c("E", "F", "G"))) return(NULL)
+    for (cand in ig_recipe_ids) {
+      if (identical(cand, rid)) next
+      if (setequal(suffix_sets[[cand]], own_suffix)) return(cand)
+    }
+    NULL
+  }
+
+  lapply(suggestions, function(s) {
+    # `s$ig_recipe_id <- NULL` would DELETE the element rather than set it
+    # (standard R list-assignment gotcha), leaving no-match entries missing
+    # the key entirely instead of carrying it as NULL. Single-bracket
+    # assignment with a wrapped list preserves a NULL-valued element so the
+    # field is always present, per the brief's "NULL when there is none".
+    s["ig_recipe_id"] <- list(find_counterpart(s$recipe_id))
+    s
+  })
+}
+
+#' Emit the single cli::cli_inform() message summarizing all suggestions
+#' for a verb call (the brief's "one message", not one per suggestion).
+#' Bullet text is pre-formatted plain text (no cli/glue `{}` markup) since
+#' recipe ids/labels are untrusted-ish data values, not literal call-site
+#' expressions. When a suggestion has an `ig_recipe_id`, one indented
+#' continuation line is appended naming the intergovernmental counterpart
+#' recipe (embedded `\n` renders as a hanging-indent continuation of the
+#' same bullet under cli, not a new bullet). Same treatment for
+#' `suppressed_amount` (uscogdata#9): only present when dollars were
+#' actually measured as excluded (an `empty_year` fire can carry them too --
+#' see `.build_suggestions()` -- so this keys off the amount, not `trigger`).
+#' @noRd
+.inform_suggestions <- function(suggestions) {
+  bullets <- vapply(suggestions, function(s) {
+    bullet <- sprintf("%s (%d-%d): %s", s$recipe_id,
+            s$available_years[1], s$available_years[2], s$hint)
+    # Only present when dollars were actually measured as excluded. An
+    # empty_year fire can carry them too -- the year had no rows AND the
+    # component was suppressed -- which is strictly more informative.
+    if (isTRUE(s$suppressed_amount > 0)) {
+      bullet <- paste0(bullet, sprintf(
+        "\n  $%s excluded from %s (%s), published as an aggregate or outside the crosswalk",
+        formatC(s$suppressed_amount, format = "f", digits = 0, big.mark = ","),
+        paste0("FY", s$suppressed_years, collapse = ", "),
+        paste(s$suppressed_codes, collapse = ", ")))
+    }
+    if (!is.null(s$ig_recipe_id)) {
+      bullet <- paste0(bullet, sprintf(
+        "\n  intergovernmental counterpart: recipe = '%s'", s$ig_recipe_id))
+    }
+    bullet
+  }, character(1))
+  cli::cli_inform(c(
+    i = "Incomplete coverage for the requested years; a harmonization recipe may fill it:",
+    stats::setNames(bullets, rep("*", length(bullets)))
+  ))
+}

R/suppression.R

@@ -0,0 +1,115 @@
+# R/suppression.R
+# Split out of R/suggestions.R (2026-08-05) to keep files under the project's
+# 400-line limit. Owns the second qualifying path for coverage signposting
+# (uscogdata#9): measuring, per government, the component dollars the
+# calling verb's own long view structurally excludes (aggregate-published,
+# or absent from summary_categories). See R/suggestions.R for the
+# orchestrator (`.build_suggestions()`) that calls this and the full
+# uscogdata#9 background.
+
+#' Measure, per (recipe, year), the component dollars this government holds
+#' that the calling verb's own long view structurally excludes.
+#'
+#' This is the second qualifying path for a suggestion (uscogdata#9). The
+#' first -- row absence -- only fires when a category returns NOTHING in a
+#' requested year, which is how Corrections behaves in the wide era. Public
+#' Welfare is the failure mode it misses: E74/E75/E77/E79 still return rows,
+#' so there is no absence to detect, while E67/E68 (aggregate-flagged 1967-
+#' 2011, and absent from `summary_categories` entirely) are dropped. The
+#' caller gets a plausible number a third too low, silently.
+#'
+#' "Structurally excluded" is decided by anti-joining the verb's REAL long
+#' view rather than restating its WHERE clause, so this stays correct if
+#' `spending_long_harmonized` / `revenue_long_harmonized` ever change. That
+#' anti-join is keyed on `item_code`, which is sound only because
+#' harmonization never renames a recipe component -- asserted by the "no
+#' recipe component is ever renamed by harmonization" test in
+#' tests/testthat/test-recipes.R.
+#'
+#' Note what this deliberately does NOT count as suppressed: a component
+#' excluded from the RESULT for scoping reasons -- because it belongs to a
+#' different `category`, or because `expenditure_concept` narrowed the
+#' subtypes -- is still present in the view, so it never fires. Suggesting a
+#' recipe is a coverage fix, not a category redefinition.
+#'
+#' `flow_prefixes` (uscogdata#9 review, finding I1) restricts the measured
+#' components to the CALLING VERB's own flow family (`c("E","F","G")` for
+#' spending, `c("T","A","U","B","C","D")` for revenue). Without this, a
+#' candidate recipe belonging to the OTHER flow family is always absent from
+#' this verb's view (by construction -- `cog_revenue()`'s view never carries
+#' an E-coded row) and so was always reported as "suppressed", fabricating a
+#' dollar claim across flow families (`cog_revenue(category = "Corrections")`
+#' claimed $3.63B excluded that `cog_spending()` reports and fully accounts
+#' for). Filtering on `LEFT(r.component_code, 1)` also drops M/L-prefixed
+#' components from measurement under `cog_spending()` (`flow_prefixes` never
+#' includes "M"/"L") -- harmless today, because a recipe's own M/L components
+#' (e.g. `corrections_ig_local_combined`'s M04/M05) are present in the view
+#' in every year they exist and so never fired as suppressed anyway, but
+#' worth recording since this filter is now the thing relied on to prevent
+#' it.
+#'
+#' @param con Active DuckDB connection.
+#' @param candidates Character vector of recipe ids to measure.
+#' @param govid Character vector of canonical_govid values.
+#' @param years Integer vector of requested years.
+#' @param long_view Name of the verb's long view, from `.select_long_view()`.
+#' @param flow_prefixes The calling verb's own flow-type prefixes (see
+#'   `.build_suggestions()`). Only recipe components whose first character is
+#'   in this set are measured.
+#' @return Tibble of `recipe_id`, `year`, `suppressed_amount` (full US
+#'   dollars), `suppressed_codes` (comma-joined, sorted). Zero rows when
+#'   nothing is suppressed.
+#' @noRd
+.suppressed_components <- function(con, candidates, govid, years, long_view,
+                                    flow_prefixes) {
+  empty <- tibble::tibble(
+    recipe_id = character(0), year = numeric(0),
+    suppressed_amount = numeric(0), suppressed_codes = character(0)
+  )
+  if (length(candidates) == 0L) return(empty)
+
+  # long_view is interpolated as a SQL IDENTIFIER, not a literal, so it can
+  # never be quoted safely. It is always internally derived from a fixed
+  # view_base, so an off-allowlist value is a programming error, not input.
+  if (!long_view %in% c("spending_long", "spending_long_harmonized",
+                        "revenue_long", "revenue_long_harmonized")) {
+    cli::cli_abort(
+      "Internal error: unexpected `long_view` {.val {long_view}}.",
+      class = "uscogdata_internal_error"
+    )
+  }
+
+  sql <- sprintf(
+    "SELECT r.recipe_id,
+            l.year,
+            SUM(l.amt) * 1000.0 AS suppressed_amount,
+            string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code)
+              AS suppressed_codes
+     FROM long l
+     JOIN harmonization_recipes r
+       ON l.item_code = r.component_code
+      AND l.year BETWEEN r.year_min AND r.year_max
+      AND (r.gov_type_scope = 'all'
+           OR (r.gov_type_scope = 'state' AND l.type = 0)
+           OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
+     WHERE r.recipe_id IN (%1$s)
+       AND l.canonical_govid IN (%2$s)
+       AND l.year IN (%3$s)
+       AND l.amt <> 0
+       AND LEFT(r.component_code, 1) IN (%5$s)
+       AND NOT EXISTS (
+         SELECT 1 FROM %4$s v
+         WHERE v.canonical_govid = l.canonical_govid
+           AND v.year = l.year
+           AND v.item_code = l.item_code
+           AND v.year IN (%3$s)              -- restated: enables partition pruning (I3a)
+           AND v.canonical_govid IN (%2$s)   -- restated: pushes the govid filter (I3a)
+       )
+     GROUP BY 1, 2
+     ORDER BY 1, 2",
+    .sql_lit_chr(candidates), .sql_lit_chr(govid),
+    paste(as.integer(years), collapse = ","), long_view,
+    .sql_lit_chr(flow_prefixes)
+  )
+  tibble::as_tibble(DBI::dbGetQuery(con, sql))
+}

R/views.R

@@ -1,11 +1,95 @@
 # R/views.R
 
+# SQL files that cannot be registered unconditionally against a v4 corpus,
+# for one of two distinct reasons -- both fail at CREATE VIEW time (DuckDB
+# resolves a view's source schema eagerly, even though it defers execution),
+# so a v4 corpus can't tolerate either unconditionally:
+#
+#  (a) Missing FILE. 33-/34-/35- read_parquet() a v5-only parquet table
+#      (harmonization_map.parquet, harmonization_recipes.parquet,
+#      series_breaks.parquet) that doesn't exist at all on a v4 corpus --
+#      "IO Error: No files found".
+#
+#  (b) Missing COLUMN. 22-/23-/25- reference `long.harmonized_code`, a
+#      column that does not exist on a v4 corpus's `long` table (harmonized
+#      space was introduced in schema v5) -- "Binder Error: Referenced
+#      column harmonized_code not found". 42-/43-/45- are on this list only
+#      because they SELECT s.* FROM the (a)/(b) views above, so they'd fail
+#      to resolve their own source view if it weren't already skipped.
+#
+# Registration is therefore gated on manifest$schema_version >= 5 for all of
+# them; verb-level *usage* of the resulting views is separately gated by
+# .resolve_basis() / .require_schema_v5().
+.harmonization_view_files <- c(
+  "22-spending_long_harmonized.sql",
+  "23-revenue_long_harmonized.sql",
+  "25-ig_long_harmonized.sql",
+  "33-harmonization_map.sql",
+  "34-harmonization_recipes.sql",
+  "35-series_breaks_pq.sql",
+  "42-spending_annotated_harmonized.sql",
+  "43-revenue_annotated_harmonized.sql",
+  "45-ig_annotated_harmonized.sql"
+)
+
+# The representation contract (cog_pipeline#64): two parquet tables that say
+# what an ABSENT cell means in a given year. Gated on manifest PRESENCE, not
+# on schema_version, because the sparsification that introduced them did not
+# bump the version -- the pre-sparsification corpus this package shipped
+# against until 2026-07-30 was already schema v6 and carried neither table.
+# Keying off the version number would therefore register a view over a file
+# that does not exist and fail at CREATE VIEW time on exactly the corpora this
+# check exists to tolerate.
+.representation_view_files <- c(
+  "36-representation.sql" = "representation.parquet",
+  "37-code_set.sql"       = "code_set.parquet"
+)
+
+# Cash and security holdings (uscogdata#25). 46- selects
+# `c.balance_subtype`, a column that arrived with cog_pipeline #76/#77 and
+# WITHOUT a schema_version bump -- so neither existing gate applies:
+# .harmonization_view_files keys on schema_version, .representation_view_files
+# on the presence of a FILE. Here the discriminator is a COLUMN on a table
+# that exists either way. CREATE VIEW resolves its source schema eagerly, so
+# on an older corpus 46- would fail at registration with "Binder Error:
+# Referenced column balance_subtype not found" rather than at query time.
+.balance_view_files <- c("26-balance_long.sql", "46-balance_annotated.sql")
+
+#' Does the mounted corpus's `summary_categories` carry `balance_subtype`?
+#' Probed against the live connection rather than the manifest, because the
+#' manifest describes files, not columns.
+#' @noRd
+.corpus_has_balance_subtype <- function(con) {
+  n <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS n FROM information_schema.columns
+     WHERE table_name = 'summary_categories'
+       AND column_name = 'balance_subtype'"
+  )$n
+  isTRUE(as.integer(n) > 0L)
+}
+
+#' Does the mounted corpus publish `file` (e.g. "code_set.parquet")?
+#' Reads the manifest's metadata list rather than stat-ing the URL, so it
+#' works identically for a local fixture and a remote share.
+#' @noRd
+.corpus_has_table <- function(manifest, file) {
+  paths <- vapply(manifest$files$metadata %||% list(),
+                  function(f) as.character(f$path %||% ""), character(1))
+  file %in% basename(paths)
+}
+
 #' Register DuckDB views from inst/sql/ SQL files
 #' @noRd
 .register_views <- function(con, url, manifest) {
   sql_dir <- system.file("sql", package = "uscogdata")
-  files   <- list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE)
+  files   <- sort(list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE))
+  schema_version <- suppressWarnings(as.integer(manifest$schema_version %||% 0L))
   for (f in files) {
+    base <- basename(f)
+    if (base %in% .harmonization_view_files && schema_version < 5L) next
+    if (base %in% names(.representation_view_files) &&
+        !.corpus_has_table(manifest, .representation_view_files[[base]])) next
+    if (base %in% .balance_view_files && !.corpus_has_balance_subtype(con)) next
     sql <- paste(readLines(f, warn = FALSE), collapse = "\n")
     sql <- gsub("\\{url\\}", url, sql, fixed = FALSE)
     DBI::dbExecute(con, sql)

README.md

@@ -19,20 +19,99 @@ package implements.
 # pak::pkg_install("gitea.civilytics.org/Civilytics/uscogdata")
 ```
 
+## Amounts are in full US dollars
+
+Every amount column this package returns — `amt_nominal`, `amt_real`,
+`amt_per_capita_nominal`, `amt_per_capita_real` — is in **full US dollars**.
+
+The raw Census source files report **thousands of dollars**, and the corpus's
+own `amt` column preserves that. The verbs multiply by 1000 on the way out, so
+you never have to. The conversion is recorded in every result:
+
+```r
+r <- cog_spending("552025209777", 2020L)
+attr(r, "provenance")$transformations$units_conversion
+#> $applied TRUE  $source_unit "$1,000s (raw Census)"  $target_unit "$USD"  $multiplier 1000
+```
+
+**Do not multiply again.** If you have read elsewhere that COG amounts are in
+`$1,000s` — true of the raw corpus, and of `cog_explorer`'s conventions doc —
+that rule does not apply to anything a `cog_*()` verb hands you. Applying it
+twice overstates every figure by 1000x, and the result looks plausible rather
+than obviously wrong.
+
 ## Configuration
 
 - `USCOGDATA_URL` — corpus root URL (public Nextcloud share, trailing slash)
 - `USCOGDATA_CACHE_DIR` — optional override for the manifest cache directory
 - `USCOGDATA_MANIFEST_TTL_SECS` — optional manifest re-fetch TTL (default 3600)
 
+## Primary vs Direct vs Total spending
+
+`cog_spending(..., expenditure_concept = c("primary", "direct", "total"))`
+controls whose spending a result counts. Concepts are defined as sets of the
+crosswalk's `spend_subtype` values — never item-code first letters, which
+cannot classify correctly (the letter `Y` alone spans revenue, expenditure,
+and balance codes):
+
+- `"primary"` (the default) is the government's own service provision:
+  current operations, capital outlay, and assistance payments.
+- `"direct"` is Census's published Direct Expenditure: `primary` plus
+  interest on debt and insurance trust benefit payments (e.g. pensions).
+- `"total"` additionally adds the intergovernmental leg — money handed to
+  other governments to spend (`M`/`L` codes plus `Q11`/`Q12`/`Q18` state
+  payments to school systems) — which is meaningful for describing one
+  government's own budget over time, but double-counts when summed across
+  governments (a state's payment to a county is the same dollar the county
+  reports as its own direct spending).
+
+**Rule of thumb: any figure that spans more than one government uses
+`primary` or `direct`.** `cog_geographic_rollup()` and `cog_peer_compare()`
+enforce this by refusing `expenditure_concept = "total"`. See
+`vignette("total-spending", package = "uscogdata")` for the full
+explanation with worked examples.
+
+## General vs Total revenue
+
+`cog_revenue(..., revenue_concept = c("general", "total"))` selects between
+Census's two published revenue concepts, again defined as crosswalk
+`revenue_subtype` sets rather than item-code prefixes:
+
+- `"general"` (the default) is Census **General Revenue**: own-source
+  (taxes, charges, miscellaneous) plus federal, state and local
+  intergovernmental aid.
+- `"total"` is Census **Total Revenue**: `general` plus utility revenue
+  (`A91`–`A94`), liquor store revenue (`A90`), and insurance trust revenue
+  (unemployment and workers' compensation `Y` codes plus the
+  employee-retirement `X` codes).
+
+The manual defines the first by subtracting the other three from the second,
+so the two are related by Census's own identity:
+
+```
+Total Revenue = General + Utility + Liquor Store + Insurance Trust
+```
+
+Two things worth knowing before switching to `"total"`:
+
+- **Utility revenue is large for cities.** Measured on the bundled fixture,
+  utility plus liquor store revenue is 15.9% of city (type 2) revenue, versus
+  1.2% for states and 1.7% for counties. `general` excludes it by definition.
+- **The employee-retirement (`X`) codes stop at FY2016**, when those systems
+  moved out of the annual finance file into the separate Annual Survey of
+  Public Pensions. A `"total"` series therefore steps down at the
+  FY2016/FY2017 seam for reasons of collection scope, not revenue (series
+  breaks `SB197`–`SB202`, in the corpus's `series_breaks` table).
+
 ## Developer notes
 
 ### Testing
 
 The package ships a bundled fixture corpus at `inst/extdata/fixture_corpus/` —
-a 3.6 MB two-year slice (2019 + 2020) of the full corpus covering all 50
-states. `tests/testthat/setup.R` automatically points `USCOGDATA_URL` at this
-fixture, so the full test suite runs offline with no network dependency:
+a 15 MB four-year slice (2011, 2012, 2019, 2020) of the full corpus covering
+all 50 states. `tests/testthat/setup.R` automatically points `USCOGDATA_URL`
+at this fixture, so the full test suite runs offline with no network
+dependency:
 
 ```r
 devtools::test()   # uses bundled fixture, no credentials required

_pkgdown.yml

@@ -3,6 +3,12 @@ template:
   bootstrap: 5
 
 reference:
+  - title: Financial data
+    desc: Spending, revenue and balance-sheet holdings for one or more governments.
+    contents:
+      - cog_spending
+      - cog_revenue
+      - cog_balances
   - title: Search & basket
     desc: Resolve place names into canonical govids.
     contents:

data-raw/regenerate_fixture_corpus.R

@@ -0,0 +1,238 @@
+# data-raw/regenerate_fixture_corpus.R
+#
+# Regenerate inst/extdata/fixture_corpus/ from a cog_pipeline publish tree.
+#
+# What this does:
+#   1. Copies each requested year's long partition as-is (byte-for-byte)
+#      from <publish_cache>/data/long/ into the fixture. Default years are
+#      c(2011L, 2012L, 2019L, 2020L): 2011/2012 straddle the wide-aggregate
+#      -> modern-leaf format boundary (the harmonization/recipe seam), and
+#      2019/2020 are the pre-existing per-capita/CPI regression anchors.
+#      Each partition is a full year (all states/govs) as published, so
+#      Broward County FL and every other previously-pinned government stay
+#      covered without any per-gov slicing logic.
+#   2. Copies every metadata parquet the publish tree ships (see
+#      .FIXTURE_METADATA_FILES) as-is. These are small cross-vintage
+#      registries, not partitioned by year, so the fixture ships the complete
+#      tables rather than a year-scoped subset. representation.parquet and
+#      code_set.parquet are what make the sparse wide era interpretable --
+#      absence means "$0" in a dense_source year and "not reported" in a
+#      sparse_source one -- so a fixture without them cannot represent the
+#      published corpus.
+#   3. Resyncs the four reference docs (data_dictionary.md,
+#      reader-specification.md, README.md, series_breaks.md) from the
+#      publish tree's docs/.
+#   4. Hand-builds manifest.json for just the files the fixture ships,
+#      following the shape of the previous fixture manifest but with
+#      schema_version bumped to whatever the source manifest reports, and
+#      freshly computed sha256 / row_count / size_bytes for every fixture
+#      file (never copied from the source manifest, since paths and byte
+#      layout can differ subtly between a full corpus and a fixture).
+#
+# This is never a manual job: run it whenever cog_pipeline publishes a new
+# corpus vintage that the fixture should track.
+#
+# Usage (from the uscogdata package root):
+#   Rscript data-raw/regenerate_fixture_corpus.R
+#   Rscript data-raw/regenerate_fixture_corpus.R /path/to/publish_cache
+#
+# Or from R:
+#   source("data-raw/regenerate_fixture_corpus.R")
+#   regenerate_fixture_corpus(publish_cache_dir = "/path/to/publish_cache")
+
+# Every metadata parquet the publish tree ships, in the order they appear in
+# the corpus manifest. Single source of truth for both the copy step and the
+# fixture manifest, so the two can never drift apart.
+.FIXTURE_METADATA_FILES <- c(
+  "canonical_alias.parquet",
+  "canonical_fips_xwalk.parquet",
+  "census_collection_coverage.parquet",
+  "code_set.parquet",
+  "harmonization_map.parquet",
+  "harmonization_recipes.parquet",
+  "lineage_events.parquet",
+  "representation.parquet",
+  "series_breaks.parquet",
+  "summary_categories.parquet"
+)
+
+regenerate_fixture_corpus <- function(
+    publish_cache_dir = file.path(
+      "..", "cog_pipeline", "_targets", "publish_cache"
+    ),
+    fixture_dir = file.path("inst", "extdata", "fixture_corpus"),
+    fixture_years = c(2011L, 2012L, 2019L, 2020L)) {
+  stopifnot(
+    requireNamespace("digest", quietly = TRUE),
+    requireNamespace("jsonlite", quietly = TRUE),
+    requireNamespace("duckdb", quietly = TRUE),
+    requireNamespace("DBI", quietly = TRUE)
+  )
+
+  publish_cache_dir <- normalizePath(publish_cache_dir, mustWork = TRUE)
+  if (!dir.exists(fixture_dir)) dir.create(fixture_dir, recursive = TRUE)
+
+  source_manifest <- jsonlite::fromJSON(
+    file.path(publish_cache_dir, "manifest.json"),
+    simplifyVector = TRUE
+  )
+
+  .copy_long_partitions(publish_cache_dir, fixture_dir, fixture_years)
+  .copy_metadata_parquets(publish_cache_dir, fixture_dir)
+  .copy_docs(publish_cache_dir, fixture_dir)
+
+  manifest <- .build_fixture_manifest(
+    fixture_dir, source_manifest, fixture_years
+  )
+  manifest_path <- file.path(fixture_dir, "manifest.json")
+  writeLines(
+    jsonlite::toJSON(manifest, auto_unbox = TRUE, pretty = TRUE, null = "null"),
+    manifest_path
+  )
+
+  size_bytes <- sum(file.info(
+    list.files(fixture_dir, recursive = TRUE, full.names = TRUE)
+  )$size)
+  message(sprintf(
+    "Fixture corpus regenerated at %s (%.2f MB total).",
+    fixture_dir, size_bytes / 1024^2
+  ))
+  invisible(manifest)
+}
+
+# Copy each requested year's partition directory (just the parquet file
+# inside it) from the publish tree into the fixture, as-is.
+#' @noRd
+.copy_long_partitions <- function(publish_cache_dir, fixture_dir, years) {
+  for (yr in years) {
+    part_rel <- file.path("data", "long", sprintf("year=%d", yr), "part-0.parquet")
+    src <- file.path(publish_cache_dir, part_rel)
+    dst <- file.path(fixture_dir, part_rel)
+    if (!file.exists(src)) {
+      stop(sprintf("Source partition missing: %s", src))
+    }
+    dir.create(dirname(dst), recursive = TRUE, showWarnings = FALSE)
+    ok <- file.copy(src, dst, overwrite = TRUE)
+    if (!ok) stop(sprintf("Failed to copy %s -> %s", src, dst))
+  }
+  invisible(NULL)
+}
+
+# Copy the full (not year-scoped) metadata tables listed in
+# .FIXTURE_METADATA_FILES.
+#' @noRd
+.copy_metadata_parquets <- function(publish_cache_dir, fixture_dir) {
+  for (f in .FIXTURE_METADATA_FILES) {
+    src <- file.path(publish_cache_dir, "data", f)
+    dst <- file.path(fixture_dir, "data", f)
+    if (!file.exists(src)) {
+      stop(sprintf("Source metadata file missing: %s", src))
+    }
+    dir.create(dirname(dst), recursive = TRUE, showWarnings = FALSE)
+    ok <- file.copy(src, dst, overwrite = TRUE)
+    if (!ok) stop(sprintf("Failed to copy %s -> %s", src, dst))
+  }
+  invisible(NULL)
+}
+
+# Resync the four reference docs shipped alongside the fixture.
+#' @noRd
+.copy_docs <- function(publish_cache_dir, fixture_dir) {
+  docs <- c(
+    "data_dictionary.md", "reader-specification.md",
+    "README.md", "series_breaks.md"
+  )
+  dst_dir <- file.path(fixture_dir, "docs")
+  dir.create(dst_dir, recursive = TRUE, showWarnings = FALSE)
+  for (f in docs) {
+    src <- file.path(publish_cache_dir, "docs", f)
+    if (!file.exists(src)) {
+      stop(sprintf("Source doc missing: %s", src))
+    }
+    ok <- file.copy(src, file.path(dst_dir, f), overwrite = TRUE)
+    if (!ok) stop(sprintf("Failed to copy doc %s", f))
+  }
+  invisible(NULL)
+}
+
+# Count rows in a parquet file via an ephemeral DuckDB connection.
+#' @noRd
+.parquet_row_count <- function(path) {
+  con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  DBI::dbGetQuery(con, sprintf(
+    "SELECT COUNT(*) AS n FROM read_parquet(%s)",
+    .sql_quote(path)
+  ))$n
+}
+
+#' @noRd
+.sql_quote <- function(x) paste0("'", gsub("'", "''", x), "'")
+
+# Hand-build manifest.json following the shape of the previous fixture
+# manifest: schema_version / built_at / pipeline_commit / fixture_note /
+# data_vintage / scope / schema / files.long_partitions / files.metadata /
+# series_breaks_ref / reader_spec_ref. Every sha256 / row_count / size_bytes
+# is freshly computed against the files actually written into fixture_dir.
+#' @noRd
+.build_fixture_manifest <- function(fixture_dir, source_manifest, years) {
+  long_partitions <- lapply(years, function(yr) {
+    rel <- file.path("data", "long", sprintf("year=%d", yr), "part-0.parquet")
+    path <- file.path(fixture_dir, rel)
+    list(
+      year       = as.integer(yr),
+      path       = gsub("\\\\", "/", rel),
+      sha256     = digest::digest(path, algo = "sha256", file = TRUE),
+      row_count  = as.integer(.parquet_row_count(path)),
+      size_bytes = as.integer(file.info(path)$size)
+    )
+  })
+
+  metadata <- lapply(.FIXTURE_METADATA_FILES, function(f) {
+    rel <- file.path("data", f)
+    path <- file.path(fixture_dir, rel)
+    list(
+      path        = gsub("\\\\", "/", rel),
+      sha256      = digest::digest(path, algo = "sha256", file = TRUE),
+      description = f
+    )
+  })
+
+  list(
+    schema_version = as.integer(source_manifest$schema_version),
+    built_at       = format(Sys.time(), "%Y-%m-%dT%H:%M:%SZ", tz = "UTC"),
+    pipeline_commit = source_manifest$pipeline_commit,
+    fixture_note = paste(
+      "Four-year (2011, 2012, 2019, 2020) fixture for uscogdata tests. Full",
+      "corpus available via USCOGDATA_URL. Regenerated from the sparsified",
+      "schema-v6 corpus: the wide era (<= FY2011) no longer stores explicit",
+      "zeros, so FY2011 absence means Census published $0 while FY2012+",
+      "absence means not reported. representation.parquet and",
+      "code_set.parquet carry that rule and ship in full, as do every other",
+      "metadata table in the publish tree. 2011/2012 straddle both the",
+      "wide-aggregate -> modern-leaf format boundary (exercised by",
+      "basis=\"harmonized\" and recipe= queries) and the dense -> sparse",
+      "representation boundary (SB194); 2019/2020 retain the prior",
+      "per-capita/CPI regression anchors. Regenerated via",
+      "data-raw/regenerate_fixture_corpus.R."
+    ),
+    data_vintage    = source_manifest$data_vintage,
+    scope           = source_manifest$scope,
+    schema          = source_manifest$schema,
+    files = list(
+      long_partitions = long_partitions,
+      metadata        = metadata
+    ),
+    series_breaks_ref = source_manifest$series_breaks_ref,
+    reader_spec_ref   = source_manifest$reader_spec_ref
+  )
+}
+
+if (identical(environment(), globalenv()) && sys.nframe() == 0L) {
+  args <- commandArgs(trailingOnly = TRUE)
+  if (length(args) >= 1L) {
+    regenerate_fixture_corpus(publish_cache_dir = args[[1]])
+  } else {
+    regenerate_fixture_corpus()
+  }
+}

inst/extdata/fixture_corpus/manifest.json

@@ -1,82 +1,116 @@
 {
-  "schema_version": 3,
-  "built_at": "2026-04-27T16:43:46Z",
-  "pipeline_commit": "899af37",
-  "fixture_note": "Two-year (2019-2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL.",
+  "schema_version": 6,
+  "built_at": "2026-07-31T00:47:27Z",
+  "pipeline_commit": "aadb46b",
+  "fixture_note": "Four-year (2011, 2012, 2019, 2020) fixture for uscogdata tests. Full corpus available via USCOGDATA_URL. Regenerated from the sparsified schema-v6 corpus: the wide era (<= FY2011) no longer stores explicit zeros, so FY2011 absence means Census published $0 while FY2012+ absence means not reported. representation.parquet and code_set.parquet carry that rule and ship in full, as do every other metadata table in the publish tree. 2011/2012 straddle both the wide-aggregate -> modern-leaf format boundary (exercised by basis=\"harmonized\" and recipe= queries) and the dense -> sparse representation boundary (SB194); 2019/2020 retain the prior per-capita/CPI regression anchors. Regenerated via data-raw/regenerate_fixture_corpus.R.",
   "data_vintage": {
-    "census_source_downloaded": "unknown",
-    "cpi_vintage": "FRED CPIAUCSL",
+    "source_vintages": {
+      "2012": "10162019",
+      "2013": "10162019",
+      "2014": "10162019",
+      "2015": "10162019",
+      "2016": "10162019",
+      "2017": "06102021",
+      "2018": "06102021",
+      "2019": "06102021",
+      "2020": "06122023",
+      "2021": "06122023",
+      "2022": "06052025",
+      "2023": "06052025"
+    },
+    "registry_rows": 148,
     "acs_vintage": "ACS 2018-2022 5-year"
   },
   "scope": {
-    "gov_types_included": [
-      0,
-      1,
-      2,
-      3
-    ],
-    "gov_types_excluded": [
-      4,
-      5
-    ],
+    "gov_types_included": [0, 1, 2, 3],
+    "gov_types_excluded": [4, 5],
     "scope_note": "v0.1 covers state, county, city/municipality, and township governments. Special districts (type 4) and school districts (type 5) are excluded pending validation in a future cycle."
   },
   "schema": {
-    "long_column_count": 24,
-    "long_columns": [
-      "fips_state",
-      "type",
-      "fips_county",
-      "govid",
-      "gov_blank",
-      "gov_name",
-      "county_name",
-      "fips_state_code",
-      "fips_county_code",
-      "fips_place_code",
-      "population",
-      "popyear",
-      "enrollment",
-      "enrollyear",
-      "function_code",
-      "sch_level_code",
-      "fiscal_year_end",
-      "srvy_year",
-      "item_code",
-      "amt",
-      "srv_data",
-      "impute_flag",
-      "is_aggregate",
-      "canonical_govid"
-    ],
+    "long_column_count": 28,
+    "long_columns": ["fips_state", "type", "fips_county", "govid", "gov_blank", "gov_name", "county_name", "fips_state_asof", "fips_county_asof", "cog_legacy_state", "cog_legacy_county", "fips_place_code", "population", "popyear", "enrollment", "enrollyear", "function_code", "sch_level_code", "fiscal_year_end", "srvy_year", "item_code", "amt", "srv_data", "impute_flag", "is_aggregate", "canonical_govid", "harmonized_code", "survey_weight"],
     "data_dictionary": "docs/data_dictionary.md"
   },
   "files": {
     "long_partitions": [
+      {
+        "year": 2011,
+        "path": "data/long/year=2011/part-0.parquet",
+        "sha256": "7848e18497080c8980a4f89c5b386205b2c5bc90db6773827ea01ab3943d16b1",
+        "row_count": 496004,
+        "size_bytes": 2202455
+      },
+      {
+        "year": 2012,
+        "path": "data/long/year=2012/part-0.parquet",
+        "sha256": "b82ac82d5e35f844b26c887445601f3748438c52c998ba4e403b025941a6f170",
+        "row_count": 1163338,
+        "size_bytes": 5929917
+      },
       {
         "year": 2019,
         "path": "data/long/year=2019/part-0.parquet",
-        "sha256": "e1c9f426c6d7d3c51d06b3a652473b987b304836619c213f019cee4887714daa",
+        "sha256": "5cbd4726dcc7d0dab5c2a05a64702e979533ae119ed0587073cd31c089e0d737",
         "row_count": 318139,
-        "size_bytes": 1424231
+        "size_bytes": 1719548
       },
       {
         "year": 2020,
         "path": "data/long/year=2020/part-0.parquet",
-        "sha256": "9b795853a848e8c955c80261b96b79630fc77394dcfb1a1ca288e2cd634053a3",
+        "sha256": "ee548fec80bf1beda844fe03916ac145f10dd34c45968407cc330ec260935f00",
         "row_count": 317500,
-        "size_bytes": 1427150
+        "size_bytes": 1722918
       }
     ],
     "metadata": [
+      {
+        "path": "data/canonical_alias.parquet",
+        "sha256": "3f617051c23a99bea322889857f7106df0c92954564afeec181df7083ee6698e",
+        "description": "canonical_alias.parquet"
+      },
       {
         "path": "data/canonical_fips_xwalk.parquet",
-        "sha256": "86e53e04a35f6f90bb74bb1a273e053392afa782d6f518e3e3da9c976d47f7af",
+        "sha256": "f98742f941269dacf8f7de5c273aa4dd4e75017a5bb70c054da35852a95a8d46",
         "description": "canonical_fips_xwalk.parquet"
       },
+      {
+        "path": "data/census_collection_coverage.parquet",
+        "sha256": "143e025616cde684da7c4442bc00d07fbd1556fabb0ea96223931b737e5d10a4",
+        "description": "census_collection_coverage.parquet"
+      },
+      {
+        "path": "data/code_set.parquet",
+        "sha256": "4cffcb0198dd51e4ff2b694050bb371a5f9965cdac12f25521cb628fb8e118a9",
+        "description": "code_set.parquet"
+      },
+      {
+        "path": "data/harmonization_map.parquet",
+        "sha256": "4cf32d0f817079ba4f28dc0ce65450d3247ebbf08d94c0c26c0d02af597bf812",
+        "description": "harmonization_map.parquet"
+      },
+      {
+        "path": "data/harmonization_recipes.parquet",
+        "sha256": "1133e9a0b02f8f34f5f936e55c5ecd596bb8a55d8425dcce76767f0f3203581c",
+        "description": "harmonization_recipes.parquet"
+      },
+      {
+        "path": "data/lineage_events.parquet",
+        "sha256": "36c16acfbe621d61010984767f1c566993b8a5f481a2c1e134c4c0a600e4502f",
+        "description": "lineage_events.parquet"
+      },
+      {
+        "path": "data/representation.parquet",
+        "sha256": "31ec328a7dd505a321b45f97aafff12e53d68a1a986f63509863035b22a4360d",
+        "description": "representation.parquet"
+      },
+      {
+        "path": "data/series_breaks.parquet",
+        "sha256": "06dcc995ff533e57cc65fa25086cc9bf83ba592c58bf7cc99269dc2576f69944",
+        "description": "series_breaks.parquet"
+      },
       {
         "path": "data/summary_categories.parquet",
-        "sha256": "60045e22bc2723318fa2cb73f8e5038250dc54d24b3447c6750dfe29035335b8",
+        "sha256": "e3b0efa00ce713b8f45829b89cfde24b55333f26101f0495df82d85997d18d8e",
         "description": "summary_categories.parquet"
       }
     ]

inst/schemas/provenance-v1.json

@@ -10,11 +10,100 @@
     "target":         { "type": "object" },
     "years":          { "type": "array", "items": { "type": "integer" } },
     "category":       { "type": ["string", "array", "null"] },
+    "basis":          { "type": ["string", "null"] },
+    "basis_note":     { "type": ["string", "null"] },
+    "expenditure_concept": {
+      "type": "string",
+      "enum": ["primary", "direct", "total"],
+      "description": "Which spending concept produced this result, defined as crosswalk spend_subtype sets (never item-code prefixes). 'primary' (the default) is the government's own service provision: operations + capital + assistance. 'direct' adds interest on debt and insurance trust benefit payments (Census's published Direct Expenditure). 'total' adds intergovernmental payments (M to local governments, L to state government, Q11/Q12/Q18 to school systems). Only 'primary' and 'direct' are valid for results combined across governments."
+    },
+    "expenditure_concept_note": {
+      "type": ["string", "null"],
+      "description": "How the intergovernmental leg was assembled; null for 'primary' and 'direct'."
+    },
+    "expenditure_concept_direct_suppressed": {
+      "type": "boolean",
+      "description": "TRUE when expenditure_concept = 'total' and at least one requested (year, category) has intergovernmental rows but NO Direct rows in this corpus (typically a legacy aggregate-only family) -- those result rows report the intergovernmental leg alone, not Direct + IG. Always FALSE for expenditure_concept = 'primary' or 'direct'. See the affected rows' `notes` for the recovering recipe, if any."
+    },
+    "revenue_concept": {
+      "type": "string",
+      "enum": ["general", "total"],
+      "description": "Which revenue concept produced this result, defined as crosswalk revenue_subtype sets (never item-code prefixes). 'general' (the default) is Census General Revenue: own_source + federal + state + local_aid. 'total' is Census Total Revenue: general plus utility, liquor store and insurance trust revenue. Census defines the first by subtracting the other three from the second (manual section 4.3). Meaningful for cog_revenue() results; spending results carry the default.",
+      "$comment": "The employee-retirement (X) codes inside insurance_trust stop at FY2016, so a 'total' series steps at the FY2016/FY2017 seam for collection-scope reasons (series breaks SB197-SB202)."
+    },
+    "harmonization":  { "type": "object" },
+    "recipe":         { "type": ["object", "null"] },
+    "suggestions": {
+      "type": "array",
+      "description": "Harmonization recipes that would fill incomplete coverage in the requested years for this government. Empty on a healthy query, on an un-scoped (category = NULL) query, on basis = 'raw', and on a recipe = query (which resolves its own coverage).",
+      "items": {
+        "type": "object",
+        "required": ["recipe_id", "label", "available_years", "hint", "ig_recipe_id",
+                     "trigger", "suppressed_amount", "suppressed_years", "suppressed_codes"],
+        "properties": {
+          "recipe_id": { "type": "string" },
+          "label": { "type": "string" },
+          "available_years": {
+            "type": "array",
+            "items": { "type": "integer" },
+            "description": "[year_min, year_max] of the recipe's component coverage."
+          },
+          "hint": { "type": "string" },
+          "ig_recipe_id": {
+            "type": ["string", "null"],
+            "description": "The intergovernmental (M/L) counterpart recipe covering the same function suffixes, or null. Never set for revenue recipes."
+          },
+          "trigger": {
+            "type": "string",
+            "enum": ["empty_year", "suppressed_component"],
+            "description": "Why this fired. 'empty_year': the result has no rows at all in a requested year. 'suppressed_component': the result HAS rows, but a component code carries dollars this government reports in the requested years that the verb's underlying long view structurally excludes -- aggregate-published, carrying no harmonized code, or absent from summary_categories. This is NOT the same thing as 'excluded from the result': a component present in the view under a different category (a scoping choice, e.g. a different `category` or a narrower `expenditure_concept`) contributes 0 and never fires. 'empty_year' wins when both apply, being the stronger claim; the suppressed_* fields are populated either way, using the same underlying-view measurement, and can be 0 even on an 'empty_year' fire."
+          },
+          "suppressed_amount": {
+            "type": "number",
+            "description": "Full US dollars this government reports, in the recipe's component codes, in the requested years, that the verb's underlying long view structurally excludes (aggregate-published, carrying no harmonized code, or absent from summary_categories) -- summed across those years. This is NOT the same quantity as 'what the result excludes': a component present in the view under a different category or a narrower `expenditure_concept` is scoped out on purpose, counts as 0 here, and is not suppression. 0 does not always mean full coverage -- see 'trigger' and 'empty_year'. May be negative where Census publishes a negative `amt` for the excluded rows."
+          },
+          "suppressed_years": {
+            "type": "array",
+            "items": { "type": "integer" },
+            "description": "The requested years contributing to suppressed_amount."
+          },
+          "suppressed_codes": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "The excluded component item codes, sorted."
+          }
+        }
+      }
+    },
     "scope":          { "type": "object" },
     "codes_summed":   { "type": "object" },
     "aggregate_fallback": { "type": ["object", "null"] },
     "transformations":{ "type": "object" },
     "series_break_refs": { "type": "array", "items": { "type": "string" } },
+    "completion": {
+      "type": "object",
+      "description": "What `complete = TRUE` filled. `applied` is FALSE on an ordinary query. `rows_filled` counts cells added to the requested grid, and `absence_means` maps each requested year to the meaning of an absent cell there ('census_zero' in a dense_source year, 'not_reported' in a sparse_source one). Filled rows carry `value_source` in the result: 'reported', 'census_zero' (amount 0 -- Census published $0), or 'not_reported' (amount NA -- unknown).",
+      "properties": {
+        "applied": { "type": "boolean" },
+        "rows_filled": { "type": "integer" },
+        "absence_means": { "type": "object" }
+      }
+    },
+    "corpus_break_refs": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Ids of catalogued series breaks whose fin_code is the literal 'ALL' -- caveats about the corpus as a whole (dollar precision across 1976/1977, imputation exclusion from 2002, the dense -> sparse representation change at 2012, the government id scheme change at 2017) rather than about one item code. Selected on the break_year window alone, so they do not depend on which codes a result contains. Disjoint from series_break_refs by construction: an entry qualifies the whole result, not one series."
+    },
+    "balance_caveats": {
+      "type": ["object", "null"],
+      "description": "Present only on cog_balances() results (null/absent for cog_spending()/cog_revenue()). `not_gaap` is always TRUE and `not_gaap_note` explains that Census holdings are gross -- no liabilities are netted -- so they are NOT comparable to a GAAP fund balance. `coverage_window` maps EVERY balance_subtype present in the mounted corpus -- not only the ones this query observed -- to its measured [min year, max year] there (never hardcoded), so a caller can see which families exist and over what span before deciding they missed one. `truncated` is the query-scoped field: it lists only the subtypes this result actually observed whose coverage_window does not fully span the requested years.",
+      "properties": {
+        "not_gaap": { "type": "boolean" },
+        "not_gaap_note": { "type": "string" },
+        "coverage_window": { "type": "object" },
+        "truncated": { "type": "array", "items": { "type": "string" } }
+      }
+    },
     "manifest":       { "type": "object" },
     "sql_query":      { "type": "string" }
   }

inst/sql/11-summary_categories.sql

@@ -0,0 +1,7 @@
+-- Category crosswalk. Numbered 11 (not with the other reference tables at
+-- 30+) because the flow views (20-25) classify by MEMBERSHIP in this table
+-- and DuckDB binds a view's sources eagerly at CREATE VIEW time, so it must
+-- already exist when they register.
+CREATE OR REPLACE VIEW summary_categories AS
+SELECT *
+FROM read_parquet('{url}data/summary_categories.parquet');

inst/sql/20-spending_long.sql

@@ -1,5 +1,22 @@
+-- Direct-side expenditure rows, classified by crosswalk MEMBERSHIP
+-- (summary_categories.category_type = 'expenditure'), never by item-code
+-- first letter: prefix Y alone spans revenue (Y01/Y02), expenditure
+-- (Y05/Y06) and balance codes, so no first-letter allowlist can route it
+-- (uscogdata#11, finding F-018). Which subtypes a query actually returns is
+-- decided per expenditure_concept in R (.verb_spendrev); this view carries
+-- every non-intergovernmental expenditure subtype: operations, capital,
+-- assistance, interest, insurance_benefits.
+--
+-- The intergovernmental subtype (M/L/Q codes) is deliberately carved out
+-- into ig_long: its legacy-era rows are published ONLY as aggregate-flagged
+-- rows, so it cannot live behind this view's NOT is_aggregate filter (see
+-- 24-ig_long.sql).
 CREATE OR REPLACE VIEW spending_long AS
 SELECT *
 FROM long
-WHERE LEFT(item_code, 1) IN ('E', 'F', 'G', 'K')
+WHERE item_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE category_type = 'expenditure'
+      AND spend_subtype <> 'intergovernmental'
+  )
   AND NOT is_aggregate;

inst/sql/21-revenue_long.sql

@@ -1,5 +1,18 @@
+-- Revenue rows, classified by crosswalk MEMBERSHIP rather than item-code
+-- first letter (see 20-spending_long.sql for why prefixes cannot work).
+--
+-- Carries EVERY revenue subtype. Which of Census's two published concepts a
+-- query actually returns is decided per revenue_concept in R
+-- (.verb_spendrev), exactly as expenditure_concept narrows spending_long:
+--   general = own_source + federal + state + local_aid   (the default)
+--   total   = general + utility + liquor_store + insurance_trust
+-- Census defines the first by subtracting the other three from the second
+-- (manual section 4.3), so both concepts need all four families present here.
 CREATE OR REPLACE VIEW revenue_long AS
 SELECT *
 FROM long
-WHERE LEFT(item_code, 1) IN ('T', 'A', 'U', 'B', 'C', 'D')
+WHERE item_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE category_type = 'revenue'
+  )
   AND NOT is_aggregate;

inst/sql/22-spending_long_harmonized.sql

@@ -0,0 +1,16 @@
+-- Harmonized-basis twin of 20-spending_long.sql: same crosswalk-membership
+-- classification, applied to harmonized_code (the code the row is folded
+-- onto) rather than the published item_code. Safe because the harmonized
+-- space is leaf-only and every harmonized_code in the corpus is a
+-- summary_categories member (verified at fixture regen; a code the
+-- crosswalk cannot classify would be silently dropped here).
+CREATE OR REPLACE VIEW spending_long_harmonized AS
+SELECT * REPLACE (harmonized_code AS item_code)
+FROM long
+WHERE NOT is_aggregate
+  AND harmonized_code IS NOT NULL
+  AND harmonized_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE category_type = 'expenditure'
+      AND spend_subtype <> 'intergovernmental'
+  );

inst/sql/23-revenue_long_harmonized.sql

@@ -0,0 +1,12 @@
+-- Harmonized-basis twin of 21-revenue_long.sql: same crosswalk-membership
+-- classification (every revenue subtype; the concept narrows in R), applied
+-- to harmonized_code rather than the published item_code.
+CREATE OR REPLACE VIEW revenue_long_harmonized AS
+SELECT * REPLACE (harmonized_code AS item_code)
+FROM long
+WHERE NOT is_aggregate
+  AND harmonized_code IS NOT NULL
+  AND harmonized_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE category_type = 'revenue'
+  );

inst/sql/24-ig_long.sql

@@ -0,0 +1,25 @@
+-- Intergovernmental expenditure rows: crosswalk spend_subtype =
+-- 'intergovernmental' (M = to local govts, L = to state govts, Q11/Q12/Q18
+-- = state payments to school systems -- uscogdata#11, finding F-017).
+--
+-- Deliberately does NOT filter `NOT is_aggregate`, unlike spending_long. In the
+-- wide era (<= FY2011) the IG families M05/M12/M47/M89/L47/L89 are published
+-- ONLY as aggregate-flagged rows -- filtering them would hide ~70% of legacy IG
+-- dollars and make Total silently collapse to Direct. This is safe because the
+-- aggregate codes and their modern leaf components are strictly year-disjoint
+-- (M47 ends 2011 / M94 starts 2012; M89 is aggregate only <= 2011 and a leaf
+-- from 2012 alongside M91-93), so no row is ever counted twice. Same argument
+-- the pipeline's recipe joins use.
+--
+-- `L--` stays excluded: it is the IG-to-state FAMILY TOTAL and genuinely
+-- rolls up the L-NN codes, so including it would double-count. The crosswalk
+-- deliberately carries no `--` family-total codes, so membership excludes it
+-- (guarded by "the IG leg never includes the L-- family total" in
+-- tests/testthat/test-expenditure-concept.R).
+CREATE OR REPLACE VIEW ig_long AS
+SELECT *
+FROM long
+WHERE item_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE spend_subtype = 'intergovernmental'
+  );

inst/sql/25-ig_long_harmonized.sql

@@ -0,0 +1,22 @@
+-- Harmonized-basis IG rows. Uses COALESCE(harmonized_code, item_code) rather
+-- than harmonized_code alone: aggregate rows carry NO harmonized_code by
+-- construction (harmonized space is leaf-only), so a plain
+-- `harmonized_code IS NOT NULL` filter would drop every legacy IG aggregate --
+-- in the bundled fixture corpus (year 2011; 2012+ all carry a harmonized_code)
+-- that is $379,016,063k across 25,688 M rows and $2,277,458k across 19,266 L
+-- rows (`SELECT year, LEFT(item_code,1), SUM(amt), COUNT(*) FROM ig_long
+-- WHERE harmonized_code IS NULL GROUP BY 1, 2`). COALESCE keeps the one real
+-- IG collapse rule (M38 -> M36, SB012, year-disjoint 1967-2011 vs 2012+)
+-- while never dropping a row.
+--
+-- Membership is checked on the published item_code (mirroring 24-ig_long.sql)
+-- rather than the COALESCEd code: every IG harmonization target (M36) is
+-- itself an IG crosswalk member, so the two are equivalent, and item_code is
+-- the column that exists on every row.
+CREATE OR REPLACE VIEW ig_long_harmonized AS
+SELECT * REPLACE (COALESCE(harmonized_code, item_code) AS item_code)
+FROM long
+WHERE item_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE spend_subtype = 'intergovernmental'
+  );

inst/sql/26-balance_long.sql

@@ -0,0 +1,22 @@
+-- Cash and security holdings, classified by crosswalk MEMBERSHIP on
+-- category_type (see 21-revenue_long.sql for why first-letter prefixes cannot
+-- do this job -- the X and Y families each span revenue, expenditure AND
+-- balance).
+--
+-- These rows are STOCKS: a balance at a point in time, not a flow over a
+-- fiscal year. Summing a stock with a flow is meaningless, which is why they
+-- live behind a third view rather than as a subtype of either money view, and
+-- why neither spending_long nor revenue_long can reach them.
+--
+-- `NOT is_aggregate` mirrors spending_long / revenue_long. The wide-era
+-- aggregate-only holdings codes (X40/X41) are deliberately outside this view;
+-- they are reachable only through the recipe path, which bypasses this filter
+-- by design (cog_pipeline/docs/phase_r_harmonization_review.md § 0.2).
+CREATE OR REPLACE VIEW balance_long AS
+SELECT *
+FROM long
+WHERE item_code IN (
+    SELECT item_code FROM summary_categories
+    WHERE category_type = 'balance'
+  )
+  AND NOT is_aggregate;

inst/sql/31-summary_categories.sql

@@ -1,3 +0,0 @@
-CREATE OR REPLACE VIEW summary_categories AS
-SELECT *
-FROM read_parquet('{url}data/summary_categories.parquet');

inst/sql/32-gov_population_yearly.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE VIEW gov_population_yearly AS
+SELECT DISTINCT
+  year,
+  canonical_govid,
+  population,
+  popyear
+FROM long
+WHERE population IS NOT NULL;

inst/sql/33-harmonization_map.sql

@@ -0,0 +1,3 @@
+CREATE OR REPLACE VIEW harmonization_map AS
+SELECT *
+FROM read_parquet('{url}data/harmonization_map.parquet');

inst/sql/34-harmonization_recipes.sql

@@ -0,0 +1,3 @@
+CREATE OR REPLACE VIEW harmonization_recipes AS
+SELECT *
+FROM read_parquet('{url}data/harmonization_recipes.parquet');

inst/sql/35-series_breaks_pq.sql

@@ -0,0 +1,3 @@
+CREATE OR REPLACE VIEW series_breaks_pq AS
+SELECT *
+FROM read_parquet('{url}data/series_breaks.parquet');

inst/sql/36-representation.sql

@@ -0,0 +1,3 @@
+CREATE OR REPLACE VIEW representation AS
+SELECT *
+FROM read_parquet('{url}data/representation.parquet');

inst/sql/37-code_set.sql

@@ -0,0 +1,3 @@
+CREATE OR REPLACE VIEW code_set AS
+SELECT *
+FROM read_parquet('{url}data/code_set.parquet');

inst/sql/42-spending_annotated_harmonized.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW spending_annotated_harmonized AS
+SELECT
+  s.*,
+  x.gov_name       AS xwalk_gov_name,
+  x.govs_type,
+  x.type_label,
+  x.fips_state     AS xwalk_fips_state,
+  x.fips_county    AS xwalk_fips_county,
+  x.fips_place,
+  x.population_acs,
+  c.category,
+  c.category_type,
+  c.spend_subtype
+FROM spending_long_harmonized s
+LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+LEFT JOIN summary_categories   c USING (item_code);

inst/sql/43-revenue_annotated_harmonized.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW revenue_annotated_harmonized AS
+SELECT
+  s.*,
+  x.gov_name       AS xwalk_gov_name,
+  x.govs_type,
+  x.type_label,
+  x.fips_state     AS xwalk_fips_state,
+  x.fips_county    AS xwalk_fips_county,
+  x.fips_place,
+  x.population_acs,
+  c.category,
+  c.category_type,
+  c.revenue_subtype
+FROM revenue_long_harmonized s
+LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+LEFT JOIN summary_categories   c USING (item_code);

inst/sql/44-ig_annotated.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW ig_annotated AS
+SELECT
+  s.*,
+  x.gov_name       AS xwalk_gov_name,
+  x.govs_type,
+  x.type_label,
+  x.fips_state     AS xwalk_fips_state,
+  x.fips_county    AS xwalk_fips_county,
+  x.fips_place,
+  x.population_acs,
+  c.category,
+  c.category_type,
+  c.spend_subtype
+FROM ig_long s
+LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+LEFT JOIN summary_categories   c USING (item_code);

inst/sql/45-ig_annotated_harmonized.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW ig_annotated_harmonized AS
+SELECT
+  s.*,
+  x.gov_name       AS xwalk_gov_name,
+  x.govs_type,
+  x.type_label,
+  x.fips_state     AS xwalk_fips_state,
+  x.fips_county    AS xwalk_fips_county,
+  x.fips_place,
+  x.population_acs,
+  c.category,
+  c.category_type,
+  c.spend_subtype
+FROM ig_long_harmonized s
+LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+LEFT JOIN summary_categories   c USING (item_code);

inst/sql/46-balance_annotated.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW balance_annotated AS
+SELECT
+  s.*,
+  x.gov_name       AS xwalk_gov_name,
+  x.govs_type,
+  x.type_label,
+  x.fips_state     AS xwalk_fips_state,
+  x.fips_county    AS xwalk_fips_county,
+  x.fips_place,
+  x.population_acs,
+  c.category,
+  c.category_type,
+  c.balance_subtype
+FROM balance_long s
+LEFT JOIN canonical_fips_xwalk x USING (canonical_govid)
+LEFT JOIN summary_categories   c USING (item_code);

man/cog_balances.Rd

@@ -0,0 +1,76 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/balances.R
+\name{cog_balances}
+\alias{cog_balances}
+\title{Cash and security holdings for one or more governments}
+\usage{
+cog_balances(
+  govid,
+  years,
+  category = NULL,
+  per_capita = FALSE,
+  adjust_to_year = NULL,
+  basis = c("harmonized", "raw"),
+  recipe = NULL
+)
+}
+\arguments{
+\item{govid}{Canonical govid(s): a character vector, or a data frame with a
+`canonical_govid` column (e.g. from [cog_gov_search()]).}
+
+\item{years}{Integer vector of fiscal years.}
+
+\item{category}{Optional character vector of categories to keep. One of
+`"Fund Balances"`, `"Insurance Trust Balances"`,
+`"Retirement System Holdings"`. There is deliberately no `subtype`
+argument: for holdings, `category` is a strict coarsening of
+`balance_subtype` (unlike the money verbs, where the two axes cross), so
+every combination would be either redundant or empty.
+`category = "Fund Balances"` is exactly the `general` family
+(`W01`/`W31`/`W61`). `balance_subtype` is returned, so a finer split is
+one `dplyr::filter()` away.}
+
+\item{per_capita}{Divide holdings by population. Note this is a **stock per
+resident** (reserves per person), which is *not* comparable to
+[cog_spending()]'s per-capita figures -- those are a flow per person.}
+
+\item{adjust_to_year}{Deflate to this year's dollars (CPI-U).}
+
+\item{basis}{Accepted for uniformity with the money verbs, but currently a
+**no-op**: `harmonization_map` carries no balance-code rows, so harmonized
+and raw space are identical for holdings. Reported in
+`provenance$basis_note`.}
+
+\item{recipe}{Optional harmonization recipe id (see [cog_recipes()]).
+`"cash_securities_z77_wide"` and `"cash_securities_z78_wide"` bridge the
+wide era to the modern one.}
+}
+\value{
+Tibble with columns `year`, `canonical_govid`, `gov_name`,
+  `balance_subtype`, `category`, `amt_nominal`, `codes_included`,
+  `aggregate_fallback`, plus optional `amt_per_capita_nominal` and
+  `pop_source` (when `per_capita = TRUE`), optional `amt_real` (when
+  `adjust_to_year` is set), and optional `amt_per_capita_real` (only when
+  **both** `per_capita = TRUE` and `adjust_to_year` are set -- there is no
+  nominal per-capita column to deflate otherwise). Amounts are full US
+  dollars.
+
+  Carries a `provenance` attribute matching
+  `inst/schemas/provenance-v1.json`, whose `balance_caveats` block reports
+  `not_gaap`, `not_gaap_note`, `coverage_window` (measured year extents for
+  every balance subtype in the mounted corpus, not only the observed ones)
+  and `truncated` (the observed subtypes whose coverage falls short of the
+  requested years). `expenditure_concept`/`revenue_concept` are `NA` --
+  holdings are a stock, not a flow, so neither concept vocabulary applies.
+}
+\description{
+Returns Census cash-and-security holdings (`category_type = "balance"`):
+fund balances, retirement system holdings and insurance trust balances.
+}
+\section{Holdings are not GAAP fund balance}{
+
+Census holdings are **gross** -- no liabilities are netted -- so a reserve
+ratio built from them overstates what is actually available. They are not
+comparable to a GAAP fund balance from an ACFR.
+}
+

man/cog_categories.Rd

@@ -7,8 +7,8 @@
 cog_categories(type = NULL, pattern = NULL)
 }
 \arguments{
-\item{type}{Either `NULL` (default, return both spending and revenue
-rows), `"spending"`, or `"revenue"`.}
+\item{type}{Either `NULL` (default, every row: expenditure, revenue and
+balance), `"spending"`, `"revenue"`, or `"balance"`.}
 
 \item{pattern}{Optional regex matched case-insensitively against the
 `category` column (e.g. `"Police"` or `"Tax"`).}
@@ -22,7 +22,15 @@ Tibble with columns `category`, `category_type`, `subtype`,
 Returns the category taxonomy exposed by the corpus's
 `summary_categories` view, grouped to one row per
 `(category, subtype)` pair. Use this to discover valid `category`
-values for [cog_spending()] / [cog_revenue()] /
+values for [cog_spending()] / [cog_revenue()] / [cog_balances()] /
 [cog_geographic_rollup()] and to audit which Census item codes feed
 each category.
 }
+\details{
+`subtype` COALESCEs the crosswalk's three subtype columns, so it carries
+`spend_subtype` on expenditure rows, `revenue_subtype` on revenue rows and
+`balance_subtype` on balance rows. Note that [cog_balances()] itself takes
+no `subtype` argument — for holdings, `category` is a strict coarsening of
+`balance_subtype` — but the value is surfaced here because it is the
+discovery surface downstream consumers build their vocabulary from.
+}

man/cog_find_peers.Rd

@@ -6,17 +6,22 @@
 \usage{
 cog_find_peers(
   target_govid,
+  year = NULL,
   same_type = TRUE,
   same_state = FALSE,
   pop_range = c(0.7, 1.3),
   is_ratio = TRUE,
-  pop_year = NULL,
-  max_peers = 10L
+  max_peers = 10L,
+  coverage = c("all", "census", "consistent")
 )
 }
 \arguments{
 \item{target_govid}{Character scalar — `canonical_govid` of the target.}
 
+\item{year}{Integer scalar. Cohort vintage. When `NULL` (default), uses the
+most recent year for which the target has an observed population in
+`gov_population_yearly`.}
+
 \item{same_type}{If `TRUE` (default) restrict peers to the target's
 `govs_type`.}
 
@@ -26,20 +31,26 @@ Default `FALSE`.}
 \item{pop_range}{Length-2 numeric vector giving lower/upper bounds.}
 
 \item{is_ratio}{If `TRUE` (default) `pop_range` is multiplied by the
-target's `population_acs` to produce absolute bounds. If `FALSE`,
+target's population at `year` to produce absolute bounds. If `FALSE`,
 `pop_range` is interpreted as absolute population counts.}
 
-\item{pop_year}{Reserved for future use (selecting ACS vintage). Currently
-the corpus has a single snapshot so this argument has no effect.}
-
 \item{max_peers}{Integer cap on the number of peers returned.}
+
+\item{coverage}{Survey-cycle handling; see [cog_peer_compare()]. Here it
+governs the cohort VINTAGE when `year` is `NULL`: `"census"` snaps to the
+most recent census year with an observed population, so a cohort is not
+built from a sample year in which most of the candidate universe is
+absent. `"consistent"` needs a year range, which cohort selection does not
+have, so it selects like `"all"` and is carried on the result as
+`attr(x, "coverage")` for [cog_peer_compare()].}
 }
 \value{
 Tibble with columns `canonical_govid`, `gov_name`, `fips_state`,
-  `population_acs`, `pop_ratio`, `rank`.
+  `population`, `pop_ratio`, `rank`. The cohort year is attached as
+  `attr(x, "cohort_year")`.
 }
 \description{
-Selects peer governments from `canonical_fips_xwalk` by combinations of
-government type, state, and population range. Peers are ordered by
-`|log(pop_ratio)|` ascending (closest to the target's population first).
+Selects peer governments by combinations of government type, state, and
+population range at a chosen `year`. Peers are ordered by `|log(pop_ratio)|`
+ascending (closest to the target's population first).
 }

man/cog_geographic_rollup.Rd

@@ -9,7 +9,9 @@ cog_geographic_rollup(
   category,
   years,
   per_capita = FALSE,
-  adjust_to_year = NULL
+  adjust_to_year = NULL,
+  expenditure_concept = c("primary", "direct", "total"),
+  coverage = c("all", "census", "consistent")
 )
 }
 \arguments{
@@ -22,17 +24,46 @@ to [cog_spending()]).}
 
 \item{years}{Integer vector of years.}
 
-\item{per_capita}{If `TRUE`, per-capita uses each layer's own population
-from `canonical_fips_xwalk.population_acs`.}
+\item{per_capita}{If `TRUE`, per-capita uses each gov's own per-year
+population from `gov_population_yearly`. Govs with missing population
+are excluded from the result.}
 
 \item{adjust_to_year}{Integer base year for CPI-U conversion, or `NULL`.}
+
+\item{expenditure_concept}{`"primary"` (default), `"direct"`, or
+`"total"` -- see [cog_spending()] for the three concepts. `"total"` is
+refused here because combining Total across multiple layers of
+government double-counts intergovernmental transfers (a state's payment
+to a school district is the same dollar the district reports as its own
+Direct spending); `"primary"` and `"direct"` combine safely.}
+
+\item{coverage}{How to handle the Census of Governments survey cycle,
+  which is a **complete census only in years ending in 2 and 7** -- every
+  other year is a sample, and the sample varies enormously (on the bundled
+  fixture, Wisconsin's 608-city universe reports 597 governments in FY2012
+  and 112 in FY2019).
+
+  * `"all"` (default) -- every unit that reported that year. Unchanged
+    behaviour, so existing code keeps working.
+  * `"census"` -- census years only. Aborts if the requested range holds
+    none, rather than silently returning nothing.
+  * `"consistent"` -- only units reporting in *every* requested year, giving
+    a balanced panel.
+
+  Regardless of mode, `provenance$coverage` always carries per-year
+  `n_units_reporting`, `n_units_expected` and `is_census_year`, and
+  `provenance$coverage_mode` records the mode. `is_census_year` is a
+  statement about the **survey calendar**, never a claim of completeness:
+  FY1967 is a census year in which only 97 of Wisconsin's 608 cities
+  report. `n_units_reporting` is the number that tells the truth.}
 }
 \value{
 Tibble with columns `year`, `layer`, `canonical_govid`, `gov_name`,
   `spend_subtype`, `category`, `amt_nominal`, optional `amt_real` /
-  `amt_per_capita_nominal` / `amt_per_capita_real`, `codes_included`,
-  `aggregate_fallback`, `scope_note`, `notes`. Carries a `provenance`
-  attribute with `verb = "cog_geographic_rollup"` and `layers`.
+  `amt_per_capita_nominal` / `amt_per_capita_real`, optional `pop_source`,
+  `codes_included`, `aggregate_fallback`, `scope_note`, `notes`. Carries a
+  `provenance` attribute with `verb = "cog_geographic_rollup"`, `layers`,
+  and `rollup$included_govids` / `rollup$excluded_govids`.
 }
 \description{
 Wraps [cog_spending()], tags each row with its layer, and attaches a
@@ -41,3 +72,11 @@ human-readable `scope_note` documenting geographic-scope caveats (e.g.
 "place portraits" that compare a city to the surrounding county and
 containing state on one set of axes.
 }
+\details{
+When `per_capita = TRUE`, rows whose government has no observed
+population in that year (`pop_source == "unavailable"`) are dropped from
+the result. The dropped govids are recorded in
+`provenance$rollup$excluded_govids`. This excludes special districts
+(gov type 4) and school districts (gov type 5) from per-capita rollups
+by design — see `vignette('population-denominators')`.
+}

man/cog_gov_search.Rd

@@ -32,8 +32,11 @@ the cross-vintage canonical-government registry. Operates in two modes:
 }
 \details{
 * **Utility mode** (single `name`, the original behavior): returns all
-  rows whose `gov_name` matches the regex case-insensitively, sorted by
-  `population_acs` descending. Useful for exploratory lookups.
+  rows whose `gov_name` contains `name` as a **literal, case-insensitive
+  substring**, sorted by `population_acs` descending. Useful for
+  exploratory lookups. Regex metacharacters in `name` are escaped, so a
+  government is findable by its own complete name even when that name
+  contains parentheses or a period.
 * **Basket mode** (`length(name) > 1`): resolves each input row to a
   single canonical govid and returns a tibble in input order, suitable
   for piping straight into [cog_spending()] / [cog_revenue()] /
@@ -45,7 +48,8 @@ the cross-vintage canonical-government registry. Operates in two modes:
 1. Filter `canonical_fips_xwalk` by `state` and (if non-NA) `type`.
 2. **Exact pass:** case-insensitive equality against `gov_name`.
    Single hit -> resolved. Multiple -> step 4.
-3. **Substring fallback:** case-insensitive regex against `gov_name`.
+3. **Substring fallback:** case-insensitive literal substring against
+   `gov_name` (metacharacters escaped).
    Single hit -> resolved (`match_method = "substring"`). Zero hits ->
    `status = "no_match"`. Multiple hits -> step 4.
 4. **Disambiguation:** if matches share one `govs_type`, pick the
@@ -58,7 +62,7 @@ inputs (`ambiguous` / `no_match`) appear only in the sidecar.
 }
 \examples{
 \dontrun{
-# Utility mode — exploratory regex lookup
+# Utility mode — exploratory substring lookup
 cog_gov_search("broward", state = "FL")
 
 # Basket mode — resolve a known cohort

man/cog_manifest.Rd

@@ -0,0 +1,18 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/manifest.R
+\name{cog_manifest}
+\alias{cog_manifest}
+\title{Return the parsed corpus manifest for the active session.}
+\usage{
+cog_manifest()
+}
+\value{
+Named list: `schema_version`, `built_at`, `pipeline_commit`,
+  `data_vintage`, `scope`, `years` (schema v5+), `schema`, `files`.
+}
+\description{
+Opens a session (connecting to the configured corpus) if none is active,
+then returns the manifest exactly as parsed from `manifest.json`. Useful
+for consumers that need the published year range (`years` block, schema
+v5+) or the partition list without issuing a data query.
+}

man/cog_peer_compare.Rd

@@ -10,7 +10,9 @@ cog_peer_compare(
   category,
   years,
   per_capita = TRUE,
-  adjust_to_year = NULL
+  adjust_to_year = NULL,
+  expenditure_concept = c("primary", "direct", "total"),
+  coverage = c("all", "census", "consistent")
 )
 }
 \arguments{
@@ -27,18 +29,81 @@ cog_peer_compare(
 population.}
 
 \item{adjust_to_year}{Integer base year for CPI-U conversion or `NULL`.}
+
+\item{expenditure_concept}{`"primary"` (default), `"direct"`, or
+`"total"` -- see [cog_spending()] for the three concepts. `"total"` is
+refused here because combining Total across peer sets counts
+intergovernmental transfers twice; `"primary"` and `"direct"` combine
+safely.}
+
+\item{coverage}{How to handle the Census of Governments survey cycle,
+  which is a **complete census only in years ending in 2 and 7** -- every
+  other year is a sample, and the sample varies enormously (on the bundled
+  fixture, Wisconsin's 608-city universe reports 597 governments in FY2012
+  and 112 in FY2019).
+
+  * `"all"` (default) -- every unit that reported that year. Unchanged
+    behaviour, so existing code keeps working.
+  * `"census"` -- census years only. Aborts if the requested range holds
+    none, rather than silently returning nothing.
+  * `"consistent"` -- only units reporting in *every* requested year, giving
+    a balanced panel.
+
+  Regardless of mode, `provenance$coverage` always carries per-year
+  `n_units_reporting`, `n_units_expected` and `is_census_year`, and
+  `provenance$coverage_mode` records the mode. `is_census_year` is a
+  statement about the **survey calendar**, never a claim of completeness:
+  FY1967 is a census year in which only 97 of Wisconsin's 608 cities
+  report. `n_units_reporting` is the number that tells the truth.
+
+  The comparison target is exempt from `"consistent"` balancing -- it is the
+  subject of the comparison, not a member of the cohort -- and the
+  `summary_*` quantiles are computed AFTER the filter, so they describe the
+  cohort actually returned. `n_units_reporting` counts peers only, against
+  the cohort size: "3 of your 15 peers reported in FY2019".}
 }
 \value{
 Tibble matching [cog_spending()]'s columns, plus a `role`
   column taking values `"target"`, `"peer"`, `"summary_p25"`,
-  `"summary_p50"`, or `"summary_p75"`, and `target_rank` (target's rank
-  among target+peers at `max(years)`, NA for other rows). Provenance
-  attribute reports `verb = "cog_peer_compare"` and `peer_count`.
+  `"summary_p50"`, or `"summary_p75"`, `target_rank` (target's rank
+  among target+peers at `max(years)`, NA for other rows), and
+  `cohort_year` (the year used to build the peer cohort, read from
+  `attr(peers, "cohort_year")`; `NA` when `peers` was a bare character
+  vector). Provenance reports `verb = "cog_peer_compare"`, `peer_count`,
+  `cohort_year`, and `cohort_govids`.
+
+  **The `summary_*` rows are per-category quantiles: they are not additive.**
+  Each one is computed **within each `(year, spend_subtype,
+  category)` cell** across the peer set, so a `summary_p50` row is *the
+  median peer's value in that one category*, not *the value of the median
+  peer's total*. The median peer for Police and the median peer for Fire
+  are usually different governments, so summing `summary_*` rows across
+  categories does not give any peer's total and misstates the band it
+  appears to describe — measured at −32.7% to +251.0% across 24 years on
+  one cohort, with a sign flip at FY2012.
+
+  Facet by `role` **and** `category` (the documented use, and what the
+  rows are built for). For a genuine "median peer's total spending" line,
+  sum each peer's own categories first and take the quantile of those
+  per-government totals:
+
+  ```r
+  library(dplyr)
+  cmp |>
+    filter(role %in% c("target", "peer")) |>
+    group_by(year, role, canonical_govid) |>
+    summarise(total = sum(amt_per_capita_real, na.rm = TRUE), .groups = "drop") |>
+    filter(role == "peer") |>
+    group_by(year) |>
+    summarise(p50 = quantile(total, 0.5, na.rm = TRUE))
+  ```
 }
 \description{
 Pulls spending for the target plus a peer set (either a
 [cog_find_peers()] result or a character vector of `canonical_govid`) and
 appends peer-distribution summary rows (`summary_p25`, `summary_p50`,
 `summary_p75`) so the result can be faceted by `role` in a single ggplot
-call.
+call. Those summary rows are quantiles **within each category**, not
+quantiles of each peer's total — see the `@return` section before summing
+them.
 }

man/cog_recipes.Rd

@@ -0,0 +1,22 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/recipes.R
+\name{cog_recipes}
+\alias{cog_recipes}
+\title{List available harmonization recipes}
+\usage{
+cog_recipes(pattern = NULL)
+}
+\arguments{
+\item{pattern}{Optional regex matched case-insensitively against
+`recipe_id` or `label`.}
+}
+\value{
+Tibble with columns `recipe_id`, `label`, `n_components`,
+  `year_min`, `year_max` (the min/max component year coverage), sorted by
+  `recipe_id`.
+}
+\description{
+Recipes are multi-code cross-vintage series (see [cog_spending()]'s
+`recipe` argument) catalogued in the corpus's `harmonization_recipes`
+table. Use this to discover valid `recipe` ids.
+}

man/cog_revenue.Rd

@@ -9,7 +9,11 @@ cog_revenue(
   years,
   category = NULL,
   per_capita = FALSE,
-  adjust_to_year = NULL
+  adjust_to_year = NULL,
+  basis = c("harmonized", "raw"),
+  recipe = NULL,
+  revenue_concept = c("general", "total"),
+  complete = FALSE
 )
 }
 \arguments{
@@ -21,17 +25,94 @@ cog_revenue(
 `summary_categories.category`), or `NULL` for all categories.}
 
 \item{per_capita}{If `TRUE`, adds `amt_per_capita_nominal` (and
-`amt_per_capita_real` when `adjust_to_year` is set) using
-`population_acs` from the canonical xwalk.}
+`amt_per_capita_real` when `adjust_to_year` is set) using the per-year
+Census F-33 population from `gov_population_yearly`. Result also gains
+a `pop_source` column with values `"census_f33"` or `"unavailable"`
+(the latter for gov types 4/5 and any row whose population is missing
+in that year).}
 
 \item{adjust_to_year}{Integer base year for CPI-U real-dollar conversion,
 or `NULL` for nominal only.}
+
+\item{basis}{`"harmonized"` (default) sums item codes through the
+cross-vintage harmonization mapping (folding series-break-affected
+codes onto a comparable target and excluding aggregate / discontinued
+rows -- see the `harmonization` block in `cog_explain()`); `"raw"`
+reproduces the pre-Phase-R2 behavior (published item codes, no
+folding). On a corpus with `schema_version < 5` (no harmonization
+tables), `basis` silently resolves to `"raw"` when left at its default
+and the resolution is recorded in the provenance; explicitly passing
+`basis = "harmonized"` on such a corpus aborts. Ignored when `recipe`
+is set (see below).}
+
+\item{recipe}{Optional harmonization recipe id (see [cog_recipes()]) for
+multi-code cross-vintage series that a 1:1 harmonized_code mapping
+can't express (e.g. a wide-era aggregate that only splits into leaf
+codes in the modern era). Mutually exclusive with `category`. The
+result's subtype column reads `"recipe"` and `category` reads the
+recipe's label. Requires `schema_version >= 5`. A recipe query bypasses
+`basis` entirely (it joins `long` directly rather than going through
+the `*_annotated`/`*_annotated_harmonized` views), so the `basis`
+argument is ignored and the result's provenance reports
+`basis = "recipe"` with an inert `harmonization` block (`applied =
+FALSE`, pointing at the `recipe` block instead) rather than a
+possibly-misleading `"harmonized"`/`"raw"` value.}
+
+\item{revenue_concept}{Which of Census's two published revenue concepts to
+  return. Concepts are defined as sets of the crosswalk's `revenue_subtype`
+  values -- never as item-code first letters, which cannot classify
+  correctly (prefix `Y` spans revenue, expenditure and balance codes, and
+  prefix `X` does the same):
+
+  * `"general"` (default) -- Census General Revenue: `own_source` +
+    `federal` + `state` + `local_aid`. The manual defines this concept by
+    subtraction (section 4.3: *"General revenue comprises all revenue
+    except that classified as liquor store, utility, or insurance trust
+    revenue"*), so utility (`A91`-`A94`), liquor store (`A90`) and
+    insurance trust revenue are all excluded.
+  * `"total"` -- Census Total Revenue: every revenue subtype, i.e.
+    `general` plus utility, liquor store, and insurance trust revenue
+    (`Y01`/`Y02`/`Y04`/`Y11`/`Y12`/`Y51`/`Y52` and the employee-retirement
+    `X01`/`X02`/`X05`/`X08`).
+
+  The two are related by Census's own identity, `Total Revenue = General +
+  Utility + Liquor Store + Insurance Trust`.
+
+  Note that the employee-retirement (`X`) codes stop at FY2016, when those
+  systems moved out of the annual finance file into the separate Annual
+  Survey of Public Pensions, so a `"total"` series steps down at the
+  FY2016/FY2017 seam for reasons that are about collection scope rather
+  than revenue (series breaks `SB197`-`SB202`).}
+
+\item{complete}{If `TRUE`, fill the requested grid so that a cell the
+  corpus does not carry still appears, labelled with **why** it is
+  missing, and add a `value_source` column to every row:
+
+  * `"reported"` — the corpus carries this cell.
+  * `"census_zero"` — dense-source year (`<= FY2011`), cell absent:
+    Census published `$0`. `amt_nominal` is `0`.
+  * `"not_reported"` — sparse-source year (`>= FY2012`), cell absent: the
+    government did not report, and the value is unknown. `amt_nominal` is
+    `NA`, **not** `0` — writing a zero there would invent data.
+
+  The grid comes from the corpus's `code_set` table, scoped to each
+  government's own type, so a county is never filled with cells only a
+  state can report. Reported rows are passed through untouched.
+
+  Defaults to `FALSE` (the historical behaviour: absent cells simply do
+  not appear). Needs a corpus published from 2026-07-29 onward, which is
+  when `representation`/`code_set` began shipping; aborts with class
+  `uscogdata_representation_unavailable` otherwise. Not available with
+  `recipe` or with `expenditure_concept = "total"` (class
+  `uscogdata_complete_unsupported`) — neither draws its cells from
+  `code_set`.}
 }
 \value{
 Tibble with columns `year`, `canonical_govid`, `gov_name`,
   `revenue_subtype`, `category`, `amt_nominal`, optional `amt_real`,
   optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
-  `codes_included`, `aggregate_fallback`, `notes`.
+  optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`,
+  and `value_source` when `complete = TRUE`.
 }
 \description{
 Mirror of [cog_spending()] for revenue categories. One row per

man/cog_spending.Rd

@@ -9,7 +9,11 @@ cog_spending(
   years,
   category = NULL,
   per_capita = FALSE,
-  adjust_to_year = NULL
+  adjust_to_year = NULL,
+  basis = c("harmonized", "raw"),
+  recipe = NULL,
+  expenditure_concept = c("primary", "direct", "total"),
+  complete = FALSE
 )
 }
 \arguments{
@@ -21,18 +25,112 @@ cog_spending(
 `summary_categories.category`), or `NULL` for all categories.}
 
 \item{per_capita}{If `TRUE`, adds `amt_per_capita_nominal` (and
-`amt_per_capita_real` when `adjust_to_year` is set) using
-`population_acs` from the canonical xwalk.}
+`amt_per_capita_real` when `adjust_to_year` is set) using the per-year
+Census F-33 population from `gov_population_yearly`. Result also gains
+a `pop_source` column with values `"census_f33"` or `"unavailable"`
+(the latter for gov types 4/5 and any row whose population is missing
+in that year).}
 
 \item{adjust_to_year}{Integer base year for CPI-U real-dollar conversion,
 or `NULL` for nominal only.}
+
+\item{basis}{`"harmonized"` (default) sums item codes through the
+cross-vintage harmonization mapping (folding series-break-affected
+codes onto a comparable target and excluding aggregate / discontinued
+rows -- see the `harmonization` block in `cog_explain()`); `"raw"`
+reproduces the pre-Phase-R2 behavior (published item codes, no
+folding). On a corpus with `schema_version < 5` (no harmonization
+tables), `basis` silently resolves to `"raw"` when left at its default
+and the resolution is recorded in the provenance; explicitly passing
+`basis = "harmonized"` on such a corpus aborts. Ignored when `recipe`
+is set (see below).}
+
+\item{recipe}{Optional harmonization recipe id (see [cog_recipes()]) for
+multi-code cross-vintage series that a 1:1 harmonized_code mapping
+can't express (e.g. a wide-era aggregate that only splits into leaf
+codes in the modern era). Mutually exclusive with `category`. The
+result's subtype column reads `"recipe"` and `category` reads the
+recipe's label. Requires `schema_version >= 5`. A recipe query bypasses
+`basis` entirely (it joins `long` directly rather than going through
+the `*_annotated`/`*_annotated_harmonized` views), so the `basis`
+argument is ignored and the result's provenance reports
+`basis = "recipe"` with an inert `harmonization` block (`applied =
+FALSE`, pointing at the `recipe` block instead) rather than a
+possibly-misleading `"harmonized"`/`"raw"` value.}
+
+\item{expenditure_concept}{Which spending concept to return. Concepts are
+  defined as sets of the crosswalk's `spend_subtype` values -- never as
+  item-code first letters, which cannot classify correctly (prefix `Y`
+  alone spans revenue, expenditure, and balance codes):
+
+  * `"primary"` (default) -- the government's own service provision:
+    `operations` + `capital` + `assistance` subtypes.
+  * `"direct"` -- Census's published Direct Expenditure: `primary` plus
+    `interest` (interest on debt) and `insurance_benefits` (insurance
+    trust benefit payments, e.g. pensions -- Census manual section
+    5.2.2.1 includes payments to retirees in Direct).
+  * `"total"` -- `direct` plus the intergovernmental leg: payments to
+    local governments (`M` codes), to the state government (`L` codes,
+    excluding the `L--` family-total rollup), and state payments to
+    school systems (`Q11`/`Q12`/`Q18`), so results gain rows with
+    `spend_subtype == "intergovernmental"`. Requires the active corpus's
+    `summary_categories` to carry M/L rows (added by cog_pipeline PR
+    #59); aborts with class `uscogdata_ig_categories_unsupported` on an
+    older corpus rather than silently under-reporting. Mutually
+    exclusive with `recipe` (a recipe already defines its own component
+    codes).
+
+  **Do not sum `"total"` results across levels of government** (e.g.
+  state + county + city): a state's `M12` payment to a school district is
+  the same dollar the district reports as its own direct `E12`, so
+  summing both double-counts it. This matters in particular with
+  [cog_geographic_rollup()], which sums across exactly that kind of
+  multi-layer government set.
+
+  In the legacy wide era (<= FY2011), some functions are published ONLY
+  as an aggregate-flagged family total (e.g. Corrections' `E04`/`E05`
+  split), which the Direct leg excludes by construction but the IG leg
+  deliberately keeps (see `inst/sql/24-ig_long.sql`). For a `"total"`
+  query, any (year, category) where this leaves intergovernmental rows
+  with NO Direct counterpart is flagged: the affected rows' `notes`
+  name the harmonization recipe that recovers the missing Direct
+  component (when one exists), and
+  `provenance$expenditure_concept_direct_suppressed` is `TRUE` -- the
+  figure in those rows is the intergovernmental leg alone, not Direct +
+  IG.}
+
+\item{complete}{If `TRUE`, fill the requested grid so that a cell the
+  corpus does not carry still appears, labelled with **why** it is
+  missing, and add a `value_source` column to every row:
+
+  * `"reported"` — the corpus carries this cell.
+  * `"census_zero"` — dense-source year (`<= FY2011`), cell absent:
+    Census published `$0`. `amt_nominal` is `0`.
+  * `"not_reported"` — sparse-source year (`>= FY2012`), cell absent: the
+    government did not report, and the value is unknown. `amt_nominal` is
+    `NA`, **not** `0` — writing a zero there would invent data.
+
+  The grid comes from the corpus's `code_set` table, scoped to each
+  government's own type, so a county is never filled with cells only a
+  state can report. Reported rows are passed through untouched.
+
+  Defaults to `FALSE` (the historical behaviour: absent cells simply do
+  not appear). Needs a corpus published from 2026-07-29 onward, which is
+  when `representation`/`code_set` began shipping; aborts with class
+  `uscogdata_representation_unavailable` otherwise. Not available with
+  `recipe` or with `expenditure_concept = "total"` (class
+  `uscogdata_complete_unsupported`) — neither draws its cells from
+  `code_set`.}
 }
 \value{
 Tibble with columns `year`, `canonical_govid`, `gov_name`,
   `spend_subtype`, `category`, `amt_nominal`, optional `amt_real`,
   optional `amt_per_capita_nominal`, optional `amt_per_capita_real`,
-  `codes_included`, `aggregate_fallback`, `notes`. Carries a `provenance`
-  attribute matching `inst/schemas/provenance-v1.json`.
+  optional `pop_source`, `codes_included`, `aggregate_fallback`, `notes`,
+  and `value_source` when `complete = TRUE`.
+  Carries a `provenance` attribute matching `inst/schemas/provenance-v1.json`,
+  whose `completion` block reports `applied`, `rows_filled`, and the
+  per-year `absence_means` rule that was applied.
 }
 \description{
 One row per `(year, canonical_govid, spend_subtype, category)`. Amounts are

specs/2026-04-29-per-year-population-denominator-design.md

@@ -0,0 +1,285 @@
+# Per-year population denominators in uscogdata
+
+**Date:** 2026-04-29
+**Status:** Design — pending implementation
+**Scope:** uscogdata 0.1 (pre-release; no version bump)
+**Related:** cog_pipeline (data dictionary updates)
+
+## Problem
+
+`uscogdata::cog_spending(per_capita = TRUE)` and `cog_revenue(per_capita = TRUE)`
+currently divide every year's nominal amount by a single static population value
+— `canonical_fips_xwalk.population_acs`, the ACS 2018-2022 5-year estimate.
+
+For a 24-year corpus (2000–2023) this introduces a systematic bias proportional
+to each government's population change over that span. Fast-growing places have
+their early-year per-capita numbers understated; shrinking places have theirs
+overstated. The bias commonly exceeds 20% and can exceed 50% for cities like
+Detroit. Provenance currently advertises this denominator explicitly, so the
+error is visible to careful users — but the default behavior produces wrong
+numbers.
+
+`cog_geographic_rollup()` has the same bug. `cog_find_peers()` /
+`cog_peer_compare()` use the same static value to define peer cohorts, which
+is defensible for matching but is no longer necessary now that per-year
+population is available.
+
+## Background — population sources
+
+| Source | What it is | Where it lives |
+|---|---|---|
+| **Census F-33 `population`** | Population value Census uses on each COG row to compute its own per-capita tables. Almost always a Population Estimates Program (PEP) estimate; sometimes lagged a year for fiscal-year alignment, recorded in `popyear` | `long.population`, `long.popyear` (per row) |
+| **PEP** (raw) | Census Bureau's official annual intercensal estimates. Distinct from F-33 because F-33 sometimes uses a lagged vintage | Not in corpus; available via tidycensus |
+| **ACS 5-year** | American Community Survey 5-year rolling average. Different methodology, includes margin of error, only available 2005-2009 onward | `canonical_fips_xwalk.population_acs` (one fixed vintage) |
+| **Decennial** | Actual count, every 10 years | Not in corpus |
+
+F-33 `population` is the right default: it's what Census itself uses, so per-
+capita results published by uscogdata reconcile with Census's own published
+tables.
+
+## Approach
+
+Use the per-row `population` already present in `long`, joined on
+`(canonical_govid, year)`. No new external data dependency. Coverage:
+
+- **Types 0–3** (state, county, city, township): observed every year by design
+- **Types 4–5** (special districts, schools): always NA — masked in
+  `cog_pipeline/R/read_modern.R` because the F-33 schema does not carry a
+  population value for these gov types
+
+Type-4 and type-5 govs return `NA` per-capita with a `pop_source = "unavailable"`
+flag and a note. No silent substitution.
+
+The architecture leaves the door open for future denominators (PEP, ACS,
+decennial) by surfacing `pop_source` as a first-class result column. Adding a
+new source later is a join change, not an API change.
+
+## Detailed design
+
+### New view: `gov_population_yearly`
+
+```sql
+-- inst/sql/32-gov_population_yearly.sql
+CREATE OR REPLACE VIEW gov_population_yearly AS
+SELECT DISTINCT
+  year,
+  canonical_govid,
+  population,
+  popyear
+FROM long
+WHERE population IS NOT NULL;
+```
+
+`SELECT DISTINCT` collapses the metadata column duplicated across each gov-year's
+item rows. A test asserts `(year, canonical_govid)` is unique to catch any
+future source-data divergence.
+
+### `cog_spending()` and `cog_revenue()`
+
+`.attach_per_capita()` (in `R/spending.R`) is rewritten to:
+
+1. Query `gov_population_yearly` for the requested govids and years.
+2. `LEFT JOIN` on `(canonical_govid, year)` so missing rows produce NA.
+3. Compute `amt_per_capita_nominal = amt_nominal / population`. NA when
+   population is NA.
+4. Drop `population` from the returned tibble (keep `pop_source` instead).
+
+Result tibble gains one new column when `per_capita = TRUE`:
+
+- `pop_source`: `"census_f33"` when a denominator was found, `"unavailable"`
+  when NA.
+
+`notes` is extended: when `pop_source == "unavailable"`, append
+`"No population denominator available for this gov type"`. The `notes` column
+is updated to concatenate multiple notes with `"; "` (it currently holds at
+most one).
+
+`amt_per_capita_real` is NA whenever `amt_per_capita_nominal` is NA.
+
+### `cog_geographic_rollup()`
+
+The current implementation does **not** sum amounts within a layer — it returns
+one row per `(year, canonical_govid, subtype, category)` tagged with its
+layer, intended for side-by-side "place portrait" comparisons (a city, the
+county containing it, the state containing both). That semantics is preserved.
+
+The only behavior change in this work is per-row exclusion when `per_capita = TRUE`:
+
+1. After `cog_spending()` returns with the per-row per-year denominator from
+   Task 3, drop rows where `pop_source == "unavailable"` so the result never
+   contains NA per-capita rows.
+2. Record the dropped `canonical_govid`s in `provenance$rollup$excluded_govids`
+   and the kept ones in `provenance$rollup$included_govids`.
+
+Documentation states explicitly: *Per-capita rollups include only governments
+observed in both the finance and population panels for the given year. Special
+districts and school districts (gov types 4 and 5) are therefore excluded from
+per-capita rollups by design.*
+
+Provenance gains:
+
+- `rollup.included_govids` — `canonical_govid`s present in the result
+- `rollup.excluded_govids` — `canonical_govid`s dropped for missing pop
+
+### `cog_find_peers()`
+
+Signature: `cog_find_peers(target_govid, year = NULL, pop_range = c(0.5, 2), ...)`
+
+- `year` is a single integer. When `NULL`, defaults to the most recent year
+  present in `gov_population_yearly` for the target.
+- Looks up target's `population` at `year`. Errors if NA, with a message
+  listing nearby years where target *is* observed.
+- Filters candidates by `gov_population_yearly.population` at the same `year`,
+  within `pop_range[1] * target_pop` and `pop_range[2] * target_pop`.
+- Orders by `|log(pop_ratio)|` ascending.
+
+Returned columns: `canonical_govid`, `gov_name`, `govs_type`, `fips_state`,
+`population`, `pop_ratio`, `rank`. The column previously named `population_acs`
+is renamed to `population`.
+
+The cohort year is attached as a tibble attribute: `attr(x, "cohort_year")`.
+
+### `cog_peer_compare()`
+
+Existing signature unchanged:
+`cog_peer_compare(target_govid, peers, category, years, per_capita = TRUE, adjust_to_year = NULL)`.
+The caller supplies `peers` (either a `cog_find_peers()` result tibble or a
+character vector of `canonical_govid`). The cohort year is implicit in
+whichever year the caller used to call `cog_find_peers()`.
+
+Behavior changes:
+
+- When `peers` is a tibble carrying `attr(peers, "cohort_year")`,
+  `cog_peer_compare()` reads it and stamps every result row with a constant
+  `cohort_year` column.
+- When `peers` is a bare character vector, `cohort_year` in the result is `NA`.
+- Provenance gets `cohort_year` (scalar or NA) and the cohort govids list.
+
+Users who want time-varying cohorts call `cog_find_peers()` per year and
+stitch the `cog_peer_compare()` results themselves — documented in the
+vignette with a worked example.
+
+### Provenance updates
+
+`provenance$transformations$per_capita` becomes:
+
+```r
+list(
+  applied = TRUE,
+  denominator_source = "Census F-33 population (per-year, from long.population)",
+  popyear_range = c(<min>, <max>),
+  pop_source_counts = list(census_f33 = N1, unavailable = N2)
+)
+```
+
+For peer compare results, additional provenance:
+
+```r
+list(
+  cohort_year = <int>,
+  cohort_govids = <character>,
+  pop_range = c(<lo>, <hi>)
+)
+```
+
+For rollup results, additional provenance:
+
+```r
+list(
+  rollup = list(
+    included_govids = <character>,
+    excluded_govids = <character>
+  )
+)
+```
+
+`R/explain.R` is updated to render the new fields.
+
+### Documentation
+
+**New vignette** `vignettes/population-denominators.Rmd`:
+
+1. The four population sources explained
+2. Why F-33 is the default — and how it reconciles with Census's own per-capita
+   tables
+3. The `popyear` quirk: Census sometimes uses a lagged estimate for fiscal-year
+   alignment. Recorded in provenance, not in the result.
+4. Worked example showing the bias from the old static-ACS approach versus
+   per-year F-33 (e.g., Detroit 2003 vs. 2023)
+5. Worked example of a rolling-cohort peer comparison built by looping
+   `cog_peer_compare()` per year
+6. Future direction: `pop_source` is structured so PEP, ACS time-series, or
+   decennial denominators can be added later without API changes
+
+**`cog_pipeline/docs/data_dictionary.md`** entry for `long.population` and
+`long.popyear`: definition, source (F-33 fixed-width files, byte ranges),
+type-4/5 masking rule, relationship to PEP.
+
+### Tests
+
+- `gov_population_yearly` returns one row per `(year, canonical_govid)` (uniqueness)
+- `cog_spending(per_capita = TRUE)` returns different denominators for
+  different years for a known gov in the fixture (use any gov whose population
+  changes between 2019 and 2020)
+- Type-4 and type-5 govids in the fixture return `pop_source = "unavailable"`
+  and `NA` per-capita with the expected note
+- `cog_geographic_rollup(per_capita = TRUE)` excludes missing-pop govs and
+  records them in provenance
+- `cog_find_peers()` defaults `year` to the most recent year for a target
+  with known population history
+- `cog_find_peers()` errors with a helpful message when target has no observed
+  population in the requested year
+- `cog_peer_compare()` defaults `cohort_year` and produces a result with a
+  constant `cohort_year` column
+- Provenance carries `denominator_source`, `popyear_range`, and
+  `pop_source_counts`
+- Regression test against a fixed govid+year showing the new per-capita value
+  differs from the old (static-ACS) by exactly the ratio of `population_acs`
+  to `long.population` for that gov-year
+
+### Migration
+
+Pre-release; no version bump. `NEWS.md` Unreleased entry:
+
+> **Per-capita denominators now use per-year Census F-33 population.**
+> Previously, `cog_spending()` and `cog_revenue()` divided all years' amounts
+> by a single ACS 2018-2022 population, producing biased per-capita values
+> for time-series. They now divide by the F-33 `population` recorded for each
+> gov-year. Type-4 (special districts) and type-5 (school districts) govs
+> return `NA` per-capita with `pop_source = "unavailable"`.
+>
+> **Peer matching now uses per-year population.** `cog_find_peers()` gains a
+> `year` argument (defaults to most recent observed year). `cog_peer_compare()`
+> gains `cohort_year`. Cohorts are still fixed for a single peer-compare call;
+> users wanting moving cohorts loop themselves.
+>
+> **Rollups exclude govs with missing population.** `cog_geographic_rollup()`
+> per-capita totals include only govs where both the finance variable and
+> population are observed in that year; excluded govids are recorded in
+> provenance.
+>
+> Returned column `population_acs` from `cog_find_peers()` is renamed to
+> `population` and reflects the cohort-year vintage.
+
+### File impact
+
+| File | Change |
+|---|---|
+| `inst/sql/32-gov_population_yearly.sql` | New |
+| `R/spending.R` (`.attach_per_capita`, `.notes_column`) | Per-year join, `pop_source`, multi-note concat |
+| `R/peers.R` (`cog_find_peers`, `cog_peer_compare`) | `year` / `cohort_year` args, query new view, column rename |
+| `R/rollup.R` | Skip-with-record for missing-pop govs |
+| `R/provenance.R` | New denominator/cohort/rollup fields |
+| `R/explain.R` | Render new fields |
+| `vignettes/population-denominators.Rmd` | New |
+| `tests/testthat/` | Per-year denominator, type-4/5, rollup exclusion, peer cohort, provenance |
+| `cog_pipeline/docs/data_dictionary.md` | Document `long.population`, `long.popyear`, masking |
+| `NEWS.md` | Unreleased entry |
+
+## Out of scope
+
+- PEP/ACS/decennial denominators — architected for, not implemented
+- `per_pupil` denominator using `long.enrollment` for type-5 — deferred
+- Covering-county fallback for type-4 — deliberately not done
+- Backfilling population for type-4/5 from any external source
+- Changes to `cog_explorer` callers — separate follow-up, after this lands

specs/2026-08-03-cog-balances-design.md

@@ -0,0 +1,281 @@
+# `cog_balances()` — a reader surface for cash and security holdings
+
+**Issue:** `uscogdata#25` requirement 2 · **Downstream:** `cog-api#26`
+**Date:** 2026-08-03 · **Status:** design, awaiting approval
+
+Requirement 1 of `uscogdata#25` (no `balance` row may reach a money verb) shipped
+with `#11`/`#12` and is asserted at both view and verb level. This spec covers
+requirement 2 only: a way to query holdings.
+
+## Decision: a verb, not an argument
+
+`cog_balances()`, parallel to `cog_spending()` / `cog_revenue()`.
+
+Holdings are a **stock** — a balance at a point in time — while the money verbs
+return **flows** over a fiscal year. The flow verbs' whole argument vocabulary
+is meaningless for a stock: `expenditure_concept` / `revenue_concept` describe
+which flows Census aggregates into a published total, and `complete=` fills a
+grid of fiscal-year cells. Overloading a money verb would put a stock behind
+arguments that all assume a flow.
+
+## The 14 codes
+
+Measured against the published corpus 2026-08-03, not transcribed from the
+issue. `year_min`/`year_max` are observed row extents.
+
+| `balance_subtype` | `category` | codes | observed years |
+|---|---|---|---|
+| `general` | Fund Balances | `W01`, `W31`, `W61` | 2012–2021 |
+| `employee_retirement` | Retirement System Holdings | `X21`, `X42`, `X44` | 1967–2016 |
+| | | `X47` | 1988–2016 |
+| | | `X30`, `Z77`, `Z78` | 2012–2016 |
+| `unemployment_trust` | Insurance Trust Balances | `Y07`, `Y08` | 1967–2023 |
+| `workers_comp_trust` | Insurance Trust Balances | `Y21` | 2012–2023 |
+| `other_insurance_trust` | Insurance Trust Balances | `Y61` | 2012–2023 |
+
+## Architecture
+
+### Two new views
+
+Mirroring the `revenue_long` / `revenue_annotated` pair exactly:
+
+- `inst/sql/26-balance_long.sql` — `category_type = 'balance' AND NOT is_aggregate`
+- `inst/sql/46-balance_annotated.sql` — joins `canonical_fips_xwalk` and
+  `summary_categories`, exposing `category`, `category_type`, `balance_subtype`
+
+`.register_views()` globs `inst/sql/*.sql` in sorted order, so both register
+with no new registration code.
+
+### A third gate list in `R/views.R`
+
+`CREATE VIEW` resolves its source schema eagerly, so a missing **column** fails
+at registration time, not at query time. `46-balance_annotated.sql` selects
+`c.balance_subtype`, which exists only on corpora built after pipeline `#76`/`#77`.
+That arrived without a `schema_version` bump, so neither existing gate applies:
+`.harmonization_view_files` keys on `schema_version`, `.representation_view_files`
+on the presence of a *file*. The discriminator here is a **column on an existing
+table**.
+
+```r
+.balance_view_files <- c("26-balance_long.sql", "46-balance_annotated.sql")
+```
+
+gated by probing `summary_categories` for `balance_subtype`, with
+`cog_balances()` erroring cleanly via `.require_balance_support()` on an older
+corpus — mirroring how `.require_schema_v5()` gates the harmonized views.
+
+### `R/balances.R` — a dedicated path, not `.verb_spendrev()`
+
+`.verb_spendrev()` is 825 lines whose concept scoping, intergovernmental leg and
+`complete=` grid are all flow-specific, and four verbs depend on it. Threading a
+third mode through it adds branching to shared code for no reuse benefit.
+
+Reused unchanged: `.build_provenance()`, `.build_series_break_refs()`,
+`.build_corpus_break_refs()`, the population join, `.inflate()`, and
+`.coerce_govid_input()`.
+
+Following the package's real two-layer convention: **view definitions** live in
+`inst/sql/`; **query construction** is inline `sprintf()` in R, as in
+`.verb_spendrev()`. (`CLAUDE.md` currently states "never inline SQL strings in R
+files", which the verb layer has never obeyed. Corrected in a separate commit —
+see Out of scope.)
+
+## Signature
+
+```r
+cog_balances(govid, years,
+             category       = NULL,   # Fund Balances | Insurance Trust Balances |
+                                      # Retirement System Holdings
+             per_capita     = FALSE,
+             adjust_to_year = NULL,
+             basis          = c("harmonized", "raw"),
+             recipe         = NULL)
+```
+
+Returns a `tbl_df` with a `provenance` attribute, like every other verb.
+
+**Absent by design:** `expenditure_concept`, `revenue_concept`, `complete`,
+and `subtype` — see below.
+
+**`per_capita` is offered.** Holdings per resident is a real measure (pension
+assets per capita, fund balance per resident). The roxygen `@param` states
+plainly that this is a *stock per resident* and is **not** comparable to
+`cog_spending()`'s per-capita figures.
+
+**`basis` is currently a no-op** — `harmonization_map` has zero balance-code
+rows, so harmonized and raw are identical for holdings. Kept for uniformity
+with the money verbs (the API would otherwise special-case), and
+`provenance$basis_note` says so outright rather than letting it look meaningful.
+
+**`recipe` ships in v1 and works.** The two holdings recipes bridge the wide era
+to the modern one:
+
+```
+cash_securities_z77_wide = X40 (1967-2011) + Z77 (2012-2023)
+cash_securities_z78_wide = X41 (1967-2011) + Z78 (2012-2023)
+```
+
+`X40`/`X41` carry ~42,700 rows that are **100% `is_aggregate = TRUE`**, so they
+are invisible to `balance_long`, which filters `NOT is_aggregate` like every
+other basis view. That is by design, not a defect:
+`cog_pipeline/docs/phase_r_harmonization_review.md` § 0.2 records that the wide
+era exposes these split families *only* as aggregates, and that the recipe join
+must therefore **not** filter `is_aggregate` — safe by construction, because
+wide rows (≤2011) are aggregate-only, modern rows (2012+) are leaf-only, and
+every component is year-scoped, so no double-count is possible. § 1 records the
+matching decision that the planned `X40→Z77` harmonization *map* rows were
+dropped and the continuity ships as recipes instead, which is why
+`harmonization_map` has no balance-code rows.
+
+The reader already implements this (`R/recipes.R`, `R/spending.R`), and it is
+verified rather than assumed: `corrections_combined` for FY2007 — a recipe whose
+wide leg `E05` is likewise aggregate-only — returns $906,743,000 against the
+live corpus. So a recipe query reaches rows the verb's own view cannot, exactly
+as intended.
+
+### No `subtype` argument: `category` is a strict coarsening
+
+`balance` is the only `category_type` in which `category` and the subtype column
+are **not** orthogonal. Measured against the published crosswalk:
+
+| `category_type` | subtypes spanning more than one category |
+|---|---|
+| expenditure | 5 of 6 (`operations`, `capital`, `interest`, `assistance`, `intergovernmental`) |
+| revenue | 1 of 7 (`own_source`) |
+| **balance** | **0 of 5** |
+
+For expenditure the two axes are a genuine cross-tab — *function* (Police, Fire)
+× *economic character* (operations, capital) — so both earn their place. For
+balance the relation is a strict tree:
+
+```
+Fund Balances              = {general}                                W01 W31 W61
+Retirement System Holdings = {employee_retirement}                    X21 X30 X42 X44 X47 Z77 Z78
+Insurance Trust Balances   = {unemployment_trust,
+                              workers_comp_trust,
+                              other_insurance_trust}                  Y07 Y08 Y21 Y61
+```
+
+Exposing both would therefore admit no useful combination. Of the 15 possible
+pairs, 3 are redundant (the subtype already implies its category) and **12 are
+guaranteed empty for every government in every year** — and an impossible query
+would fail by returning an empty tibble, which reads as "this government holds
+none" rather than "you asked a contradiction."
+
+Dropping `subtype` also keeps the verb aligned with the rest of the package: no
+uscogdata verb exposes a subtype argument. `subtype_col` is internal plumbing in
+`.verb_spendrev()`, and the API layers its own `subtype` row filter on top
+(`api/R/handlers_governments.R`). `cog-api#26` can do exactly that for
+`/balances`.
+
+`#25`'s hard requirement is still met — `category = "Fund Balances"` *is* the
+`general` family, precisely `W01`/`W31`/`W61`, in one filter. The only loss is
+isolating one of the three insurance funds in a single argument;
+`balance_subtype` remains a returned column, so that is one `dplyr::filter()`
+away.
+
+## Caveat surfacing
+
+`provenance$balance_caveats`, always present, plus one `cli_inform()` per
+session per caveat class when a query actually touches an affected family or
+year. Structured so `cog-api#26` can forward the fields verbatim.
+
+Verified against `series_breaks.csv`, not assumed:
+
+| # | Caveat | Covered by existing machinery? |
+|---|---|---|
+| 1 | Gross holdings, **not GAAP fund balance**; no liabilities netted | No — a constant, new field `not_gaap = TRUE` |
+| 2 | `W` is FY2012–2021 only | No — new `coverage_window`, **computed** from the corpus |
+| 3 | `X`/`Z` holdings end FY2016 | **Not yet.** No `series_breaks` row exists at 2016/2017 for `Z77`/`Z78`/`X30`. Reader surfaces it via `coverage_window`; flows through `series_break_refs` once the upstream entry lands (see Out of scope) |
+| 4 | `X40`/`X41` book → market at FY2002 | **Yes**, via `SB195`/`SB196` on `fin_code` `X40`/`X41`, under **two** conditions: a `recipe` query (the only path that observes those codes) **and** a year span that crosses FY2002. Asserted in the tests rather than assumed |
+
+On caveat 4's second condition: `.build_series_break_refs()` matches
+`break_year BETWEEN min(years) AND max(years)`, so a request spanning only
+2011–2012 does **not** surface `SB195`. That is correct, not a gap — such a
+series sits entirely after the change, on one consistent basis, and flagging a
+break it never crosses would be noise. The same rule is applied deliberately in
+`.build_corpus_break_refs()`. An earlier draft of this row omitted the span
+condition and overclaimed.
+
+`coverage_window` is derived per observed subtype family from the corpus, never
+hardcoded, so it stays correct as the corpus grows.
+
+`series_break_refs` and `corpus_break_refs` are otherwise populated by the
+existing code-driven builders and need no change.
+
+## Testing
+
+New `tests/testthat/test-balances.R`. The bundled fixture covers all four
+fixture years — `W` in 2012/2019/2020, the `X`/`Z` family in 2011/2012, `Y`
+throughout — so every test below runs offline.
+
+- **Inverse guard.** No flow code ever appears in `cog_balances()`, complementing
+  the already-asserted forward guard. Absence is verified against the raw corpus
+  via `read_parquet` on `data/long`, never through the verb that creates it.
+- **FY2016 seam.** The `X`/`Z` family is present in 2012 and absent in 2019;
+  `coverage_window` reports the termination and the console message fires once.
+- **Caveats.** `not_gaap` is always `TRUE`; `coverage_window` matches the
+  measured table above; the FY2002 valuation caveat fires only when the year
+  range crosses 2002 *and* touches `employee_retirement`.
+- **`per_capita`.** `amt_per_capita_nominal == amt_nominal / population`.
+- **`category = "Fund Balances"` is the `general` family.** Returns exactly
+  `W01`/`W31`/`W61` and nothing else — `#25`'s one-filter requirement, asserted
+  rather than assumed.
+- **The hierarchy holds.** Every `balance_subtype` in the crosswalk maps to
+  exactly one `category`. Asserted against the crosswalk so that an upstream
+  change breaking the tree — which would silently make `category` lossy —
+  fails here rather than in a user's analysis.
+- **`recipe` bridges the wide era.** `cash_securities_z77_wide` returns the
+  `X40` leg for a pre-2012 year, proving the aggregate-only wide rows are
+  reached — the property `phase_r_harmonization_review.md` § 0.2 depends on. A
+  regression here would silently truncate a 45-year series to five.
+- **`SB195`/`SB196` reach the user on that path.** A `recipe` query spanning
+  FY2002 carries both in `provenance$series_break_refs`, so the book → market
+  basis change is disclosed wherever `X40`/`X41` are actually observed.
+- **Gating.** `.require_balance_support()` errors cleanly on a corpus whose
+  `summary_categories` lacks `balance_subtype`.
+
+## Out of scope, tracked separately
+
+1. **Pipeline issue (new), non-blocking.** Catalogue the FY2016 termination of
+   the seven holdings codes in `series_breaks.csv`. There is currently **no**
+   entry at 2016/2017 for `Z77`/`Z78`/`X30`, although
+   `docs/phase_r_harmonization_review.md` § 2 identified the gap and recommended
+   exactly this — *"candidate new `series_breaks.csv` entries (recommend
+   `with_caution` documentation rows, no map action)"*. The follow-through never
+   happened. `SB197`–`SB202` set the precedent, giving the analogous X-flow
+   codes `coverage_restricted` + `with_caution` at 2017; `with_caution` is also
+   what keeps this out of the `joinable = "no"` identity-change rule, which
+   would otherwise oblige a harmonization-map row.
+
+   Verify the break corpus-wide and census-to-census before writing the rows.
+   `cog_balances()` does not wait on this — caveat 3 is covered reader-side by
+   `coverage_window` meanwhile, and the entry simply adds a second, catalogued
+   signpost when it lands.
+
+   **Superseded:** an earlier draft of this spec proposed adding
+   `summary_categories` rows for `X40`/`X41` and treated `recipe=` as blocked.
+   Both were wrong. `X40`/`X41` are deliberately aggregate-only per
+   `phase_r_harmonization_review.md` § 0.2, the dropped harmonization-map rows
+   are the documented § 1 decision, and the recipe path reaches them by design.
+2. **`cog-api#26`.** Adds `/balances` in all three required places — handler,
+   `param_contract`, and the `plumber.R` route signature. Lands after this.
+
+   **Two contract facts the API must carry forward**, both settled during
+   implementation and easy to get wrong from the outside:
+
+   - `provenance$balance_caveats$coverage_window` is **corpus-scoped, not
+     result-scoped**. It reports the observed year extent of *every* balance
+     subtype in the corpus, not only the subtypes a given query returned — so a
+     `category = "Fund Balances"` query still returns all five windows. That is
+     deliberate: the windows describe what the corpus holds, which is what a
+     consumer needs in order to know what it did *not* ask for. The sibling
+     field `truncated` is the result-scoped one. Documented in
+     `inst/schemas/provenance-v1.json` and mutation-guarded against silent
+     inversion.
+   - `balance_caveats` appears **only** on `cog_balances()` results. It is
+     absent from `cog_spending()`/`cog_revenue()` provenance, and the schema
+     says so — an API layer that assumes it is universal will read `NULL`.
+3. **`uscogdata/CLAUDE.md` refresh.** Separate commit. It is stale: it claims 7
+   SQL views (there are 21), 181 tests (716), a two-year fixture (four years),
+   and a "never inline SQL" rule the verb layer does not follow.

tests/testthat/helper-fixture.R

@@ -6,6 +6,33 @@ fixture_corpus_path <- function() {
   if (nzchar(p)) paste0(p, "/") else ""
 }
 
+# Path to a file in the SOURCE tree (README.md, man/*.Rd, vignettes/*.Rmd),
+# or "" when it isn't there.
+#
+# Tests that assert on documentation content have to read the sources, and the
+# sources only exist when the suite runs from a checkout. Under R CMD check the
+# suite runs from the INSTALLED package, where man/ and vignettes/ are not
+# shipped and `../../README.md` does not resolve -- so those tests must skip
+# rather than error. CI runs testthat::test_local() from the checkout BEFORE
+# rcmdcheck, so the assertions are still enforced on every push; this only
+# stops them from failing a context that structurally cannot satisfy them.
+source_tree_path <- function(...) {
+  p <- testthat::test_path("..", "..", ...)
+  if (file.exists(p)) p else ""
+}
+
+# Skip unless every named source file is present (see source_tree_path()).
+skip_if_no_source_tree <- function(...) {
+  paths <- vapply(list(...), function(rel) do.call(source_tree_path, as.list(rel)),
+                  character(1))
+  missing <- vapply(paths, function(p) !nzchar(p), logical(1))
+  testthat::skip_if(
+    any(missing),
+    "package source tree not available (running against the installed package)"
+  )
+  invisible(paths)
+}
+
 # Skip a test if no corpus is reachable (bundled fixture or explicit remote URL).
 skip_if_no_corpus <- function() {
   p <- fixture_corpus_path()
@@ -26,3 +53,137 @@ with_fixture_corpus <- function(code) {
   }, add = TRUE)
   force(code)
 }
+
+# Copy the bundled fixture to a temp dir with manifest.json's schema_version
+# patched to `version`, then run `code` against it with a clean session
+# (mirrors with_fixture_corpus()). Used to exercise the v4/v5 dual-accept
+# path without a second physical fixture tree: a real v4 corpus has no
+# harmonization_map/harmonization_recipes/series_breaks parquet files, but
+# .register_views() only *reads* those when schema_version >= 5 (see
+# R/views.R), so a doctored copy of the (v5) bundled fixture with the
+# manifest's schema_version knocked down to 4 is a faithful stand-in.
+with_doctored_schema_version <- function(version, code) {
+  src <- fixture_corpus_path()
+  tmp <- withr::local_tempdir(.local_envir = parent.frame())
+  file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
+
+  manifest_path <- file.path(tmp, "manifest.json")
+  m <- jsonlite::fromJSON(manifest_path, simplifyVector = FALSE)
+  m$schema_version <- as.integer(version)
+  writeLines(
+    jsonlite::toJSON(m, auto_unbox = TRUE, pretty = TRUE, null = "null"),
+    manifest_path
+  )
+
+  old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
+  uscogdata:::cog_close()
+  Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
+  on.exit({
+    uscogdata:::cog_close()
+    if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
+  }, add = TRUE)
+  force(code)
+}
+
+# Copy the bundled fixture to a temp dir with representation.parquet and
+# code_set.parquet removed (and dropped from the manifest's metadata list),
+# then run `code` against it. Models a corpus published BEFORE sparsification:
+# schema_version is left alone deliberately, because it was never bumped for
+# that change -- the pre-sparsification fixture this package shipped until
+# 2026-07-30 was schema v6 and carried neither table. Presence in the manifest
+# is therefore the only honest signal, and this helper is what proves the
+# package keys off it rather than off the version number.
+with_corpus_missing_representation <- function(code) {
+  src <- fixture_corpus_path()
+  tmp <- withr::local_tempdir(.local_envir = parent.frame())
+  file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
+
+  dropped <- c("representation.parquet", "code_set.parquet")
+  file.remove(file.path(tmp, "data", dropped))
+
+  manifest_path <- file.path(tmp, "manifest.json")
+  m <- jsonlite::fromJSON(manifest_path, simplifyVector = FALSE)
+  m$files$metadata <- Filter(
+    function(f) !basename(f$path) %in% dropped, m$files$metadata
+  )
+  writeLines(
+    jsonlite::toJSON(m, auto_unbox = TRUE, pretty = TRUE, null = "null"),
+    manifest_path
+  )
+
+  old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
+  uscogdata:::cog_close()
+  Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
+  on.exit({
+    uscogdata:::cog_close()
+    if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
+  }, add = TRUE)
+  force(code)
+}
+
+# Copy the bundled fixture to a temp dir with summary_categories.parquet
+# rewritten to drop every M/L (intergovernmental) row, then run `code`
+# against it with a clean session (mirrors with_fixture_corpus()/
+# with_doctored_schema_version()). Models a real pre-cog_pipeline-PR#59
+# corpus: the 66 M/L category rows shipped with NO schema_version bump (see
+# C2 in the expenditure-concept review), so schema_version is left
+# untouched here -- only the category data itself is rolled back.
+with_corpus_missing_ig_categories <- function(code) {
+  src <- fixture_corpus_path()
+  tmp <- withr::local_tempdir(.local_envir = parent.frame())
+  file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
+
+  cats_path <- file.path(tmp, "data", "summary_categories.parquet")
+  filtered_path <- file.path(tmp, "data", "summary_categories_filtered.parquet")
+  write_con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(write_con, shutdown = TRUE), add = TRUE)
+  DBI::dbExecute(write_con, sprintf(
+    "COPY (SELECT * FROM read_parquet(%s) WHERE LEFT(item_code, 1) NOT IN ('M', 'L'))
+     TO %s (FORMAT PARQUET)",
+    uscogdata:::.sql_lit_chr(cats_path), uscogdata:::.sql_lit_chr(filtered_path)
+  ))
+  file.remove(cats_path)
+  file.rename(filtered_path, cats_path)
+
+  old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
+  uscogdata:::cog_close()
+  Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
+  on.exit({
+    uscogdata:::cog_close()
+    if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
+  }, add = TRUE)
+  force(code)
+}
+
+# Copy the bundled fixture to a temp dir with summary_categories.parquet
+# rewritten to DROP the balance_subtype column, then run `code` against it.
+# Models a corpus published before cog_pipeline #76/#77. schema_version is
+# left untouched deliberately: that change shipped without a version bump, so
+# column presence is the only honest signal -- this helper is what proves the
+# package keys off it. Mirrors with_corpus_missing_ig_categories().
+with_corpus_missing_balance_subtype <- function(code) {
+  src <- fixture_corpus_path()
+  tmp <- withr::local_tempdir(.local_envir = parent.frame())
+  file.copy(list.files(src, full.names = TRUE), tmp, recursive = TRUE)
+
+  cats_path <- file.path(tmp, "data", "summary_categories.parquet")
+  filtered_path <- file.path(tmp, "data", "summary_categories_filtered.parquet")
+  write_con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(write_con, shutdown = TRUE), add = TRUE)
+  DBI::dbExecute(write_con, sprintf(
+    "COPY (SELECT * EXCLUDE (balance_subtype) FROM read_parquet(%s))
+     TO %s (FORMAT PARQUET)",
+    uscogdata:::.sql_lit_chr(cats_path), uscogdata:::.sql_lit_chr(filtered_path)
+  ))
+  file.remove(cats_path)
+  file.rename(filtered_path, cats_path)
+
+  old_url <- Sys.getenv("USCOGDATA_URL", unset = NA)
+  uscogdata:::cog_close()
+  Sys.setenv(USCOGDATA_URL = paste0(tmp, "/"))
+  on.exit({
+    uscogdata:::cog_close()
+    if (is.na(old_url)) Sys.unsetenv("USCOGDATA_URL") else Sys.setenv(USCOGDATA_URL = old_url)
+  }, add = TRUE)
+  force(code)
+}

tests/testthat/helper-walkthrough-raw.R

@@ -0,0 +1,44 @@
+# Helper for the Madison-walkthrough finding tests (uscogdata #11-#16).
+#
+# Those tests all assert something about what a `cog_*` verb includes or
+# excludes. The expected amounts must therefore come from the RAW corpus, never
+# from the verb under test: verifying an absence through the filter that creates
+# it proves nothing. `wt_raw_*()` opens its own DuckDB connection straight onto
+# the corpus's `long` parquet partitions, bypassing uscogdata's SQL views (and
+# therefore its `flow_prefixes` filtering) entirely.
+
+wt_corpus_glob <- function() {
+  url <- Sys.getenv("USCOGDATA_URL")
+  if (!nzchar(url)) testthat::skip("USCOGDATA_URL is not set")
+  paste0(sub("/$", "", url), "/data/long/**/*.parquet")
+}
+
+wt_raw_query <- function(sql) {
+  con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  DBI::dbGetQuery(con, sql)
+}
+
+# Sum of `amt` (in $1,000s, as the corpus stores it) for one government-year,
+# restricted either to an explicit set of item codes or to a set of first-letter
+# prefixes. Aggregate rows are excluded, matching every published verb.
+wt_raw_amt <- function(govid, year, codes = NULL, prefixes = NULL) {
+  stopifnot(xor(is.null(codes), is.null(prefixes)))
+  filter_sql <- if (!is.null(codes)) {
+    paste0("item_code IN (", paste0("'", codes, "'", collapse = ", "), ")")
+  } else {
+    paste0("LEFT(item_code, 1) IN (", paste0("'", prefixes, "'", collapse = ", "), ")")
+  }
+  out <- wt_raw_query(paste0(
+    "SELECT COALESCE(SUM(amt), 0) AS amt FROM read_parquet('", wt_corpus_glob(), "') ",
+    "WHERE canonical_govid = '", govid, "' AND year = ", year,
+    " AND NOT is_aggregate AND ", filter_sql
+  ))
+  out$amt[[1]]
+}
+
+# The item codes a verb reports having summed, flattened out of the
+# comma-separated `codes_included` column.
+wt_codes_included <- function(df) {
+  sort(unique(trimws(unlist(strsplit(stats::na.omit(df$codes_included), ",")))))
+}

tests/testthat/test-adjust.R

@@ -41,3 +41,10 @@ test_that(".inflate preserves NA amounts", {
   expect_true(is.na(result[2]))
   expect_false(any(is.na(result[c(1, 3)])))
 })
+
+test_that("bundled CPI covers the full 1967+ corpus era through this year", {
+  cpi <- .cpi_table()
+  expect_lte(min(cpi$year), 1967L)
+  expect_gte(max(cpi$year), as.integer(format(Sys.Date(), "%Y")))
+  expect_false(any(is.na(cpi$cpi)))
+})

tests/testthat/test-amount-units-documented.R

@@ -0,0 +1,52 @@
+# Madison walkthrough audit -- finding F-004. Tracked as uscogdata#15.
+# See docs/walkthroughs/FINDINGS.md in cog_explorer.
+#
+# The raw Census files report thousands of dollars; this package multiplies by
+# 1000 and returns full US dollars. That is the friendlier choice and is not
+# wrong -- but cog_explorer's CLAUDE.md states "All raw `amt` values are in
+# $1,000s", so a reader who applies that rule to amt_nominal overstates every
+# figure by 1000x, and gets a plausible-looking number rather than an obvious
+# error. The audit rates this the highest-consequence definitional gap it found.
+#
+# Deliberately NOT asserted here: man/cog_spending.Rd and man/cog_revenue.Rd,
+# which ALREADY carry the statement in their @return sections (verified
+# 2026-07-29), as does cog-api's data-dictionary.md (since 2b71b41). The gap is
+# in the surfaces a reader meets first and in cog_explorer's own conventions
+# doc -- see uscogdata#15 for the full surface-by-surface table and for the two
+# secondary tasks (cog_explorer/CLAUDE.md, which has no git remote, and
+# cog-api's llms.txt, which is silent on units).
+
+test_that("returned amounts are documented as full US dollars where readers meet the package", {
+
+  # README and vignettes ship only in the source tree, not in the installed
+  # package, so these assertions cannot run under R CMD check -- CI's earlier
+  # testthat::test_local() step is what enforces them. See
+  # skip_if_no_source_tree() in helper-fixture.R.
+  docs <- skip_if_no_source_tree(
+    "README.md",
+    c("vignettes", "total-spending.Rmd"),
+    c("vignettes", "population-denominators.Rmd")
+  )
+
+  says_units <- function(path) {
+    txt <- paste(readLines(path, warn = FALSE), collapse = " ")
+    grepl("full US dollars|full U\\.S\\. dollars", txt, ignore.case = TRUE) &&
+      grepl("\\$1,000s|thousands of dollars", txt, ignore.case = TRUE)
+  }
+
+  for (path in docs) expect_true(says_units(path))
+
+  # Pin the documented claim to the actual behaviour, so the two cannot drift.
+  # The expected raw amount is read straight from the corpus's parquet
+  # partitions -- never through cog_spending(), which is the thing being
+  # described. Madison FY2020: E/F/G = 623,347 ($1,000s) -> $623,347,000.
+  raw_thousands <- wt_raw_amt("552025209777", 2020L, prefixes = c("E", "F", "G"))
+  expect_equal(raw_thousands, 623347)
+
+  returned <- cog_spending(govid = "552025209777", years = 2020L)
+  expect_equal(sum(returned$amt_nominal), raw_thousands * 1000)
+
+  units <- attr(returned, "provenance")$transformations$units_conversion
+  expect_true(units$applied)
+  expect_equal(units$multiplier, 1000)
+})

tests/testthat/test-balances.R

@@ -0,0 +1,539 @@
+test_that("balance views register and carry only balance codes", {
+  skip_if_no_corpus()
+  con <- cog_open()
+  on.exit(cog_close())
+
+  views <- DBI::dbGetQuery(con,
+    "SELECT table_name FROM information_schema.tables
+     WHERE table_schema = 'main' AND table_type = 'VIEW'"
+  )$table_name
+  expect_true(all(c("balance_long", "balance_annotated") %in% views))
+
+  # Every item_code in balance_long is a category_type = 'balance' member.
+  leak <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS n FROM balance_long
+     WHERE item_code NOT IN (
+       SELECT item_code FROM summary_categories WHERE category_type = 'balance')"
+  )$n
+  expect_identical(as.integer(leak), 0L)
+
+  # balance_annotated exposes the subtype column the verb groups on.
+  cols <- DBI::dbGetQuery(con,
+    "SELECT column_name FROM information_schema.columns
+     WHERE table_name = 'balance_annotated'"
+  )$column_name
+  expect_true(all(c("category", "category_type", "balance_subtype") %in% cols))
+})
+
+test_that("inst/sql/26-balance_long.sql enforces NOT is_aggregate (real SQL text, synthetic parquet)", {
+  # Every category_type = 'balance' item_code in the bundled fixture has
+  # is_aggregate = FALSE for every row of every year -- there is no real row
+  # that would be excluded ONLY by the `AND NOT is_aggregate` predicate. An
+  # assertion against the live fixture (`WHERE is_aggregate` returns 0) is
+  # therefore vacuous: it passes identically whether or not the view's
+  # predicate is present. As with the 22-/23- and 24-/25- tests above, this
+  # reads the real inst/sql/26-balance_long.sql text off disk and executes it
+  # -- plus its 10-long.sql / 11-summary_categories.sql dependencies -- against
+  # a synthetic hive-partitioned parquet tree that DOES contain an aggregate
+  # row under a real balance item_code (W01), so a regression that drops the
+  # predicate changes which rows survive.
+  skip_if_no_corpus()
+
+  tmp <- withr::local_tempdir()
+  part_dir <- file.path(tmp, "data", "long", "year=2004")
+  dir.create(part_dir, recursive = TRUE)
+  part_path <- file.path(part_dir, "part-0.parquet")
+
+  write_con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(write_con, shutdown = TRUE), add = TRUE)
+  DBI::dbExecute(write_con, sprintf("
+    COPY (
+      SELECT * FROM (VALUES
+        ('bal-A', 'W01', 100,    false), -- control: ordinary balance row, survives
+        ('bal-B', 'W01', 999999, true)   -- excluded ONLY by `NOT is_aggregate`
+      ) AS t(canonical_govid, item_code, amt, is_aggregate)
+    ) TO %s (FORMAT PARQUET)
+  ", uscogdata:::.sql_lit_chr(part_path)))
+
+  DBI::dbExecute(write_con, sprintf("
+    COPY (
+      SELECT * FROM (VALUES
+        ('W01', 'Fund Balances', 'balance', NULL, NULL, 'general')
+      ) AS t(item_code, category, category_type, spend_subtype, revenue_subtype, balance_subtype)
+    ) TO %s (FORMAT PARQUET)
+  ", uscogdata:::.sql_lit_chr(file.path(tmp, "data", "summary_categories.parquet"))))
+
+  sql_dir <- system.file("sql", package = "uscogdata")
+  .read_view_sql <- function(filename) {
+    txt <- paste(readLines(file.path(sql_dir, filename), warn = FALSE), collapse = "\n")
+    gsub("\\{url\\}", paste0(tmp, "/"), txt, fixed = FALSE)
+  }
+
+  con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  DBI::dbExecute(con, .read_view_sql("10-long.sql"))
+  DBI::dbExecute(con, .read_view_sql("11-summary_categories.sql"))
+  DBI::dbExecute(con, .read_view_sql("26-balance_long.sql"))
+
+  rows <- DBI::dbGetQuery(con,
+    "SELECT canonical_govid, item_code, amt FROM balance_long ORDER BY canonical_govid"
+  )
+  expect_equal(nrow(rows), 1L)
+  expect_equal(rows$canonical_govid, "bal-A")
+  expect_equal(rows$amt, 100)
+})
+
+test_that("balance views are skipped on a corpus without balance_subtype", {
+  skip_if_no_corpus()
+  with_corpus_missing_balance_subtype({
+    con <- cog_open()
+    on.exit(cog_close())
+    views <- DBI::dbGetQuery(con,
+      "SELECT table_name FROM information_schema.tables
+       WHERE table_schema = 'main' AND table_type = 'VIEW'"
+    )$table_name
+    # Registration must SKIP them, not error -- an older corpus stays usable.
+    expect_false(any(c("balance_long", "balance_annotated") %in% views))
+    expect_true("revenue_long" %in% views)
+
+    # ...and calling the verb on such a corpus must hit
+    # .require_balance_support()'s curated abort (spec § Testing: "Gating"),
+    # not a DuckDB binder error naming a view that was never registered.
+    # Asserted on the CLASS: removing the guard still errors, so a bare
+    # expect_error() would pass on the regression.
+    expect_error(
+      cog_balances("550000227544", 2019),
+      class = "uscogdata_no_balance_support"
+    )
+  })
+})
+
+test_that("cog_balances returns holdings for a government that has them", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", 2019)
+    expect_s3_class(r, "tbl_df")
+    expect_true(nrow(r) > 0L)
+    expect_true(all(c("year", "canonical_govid", "gov_name", "balance_subtype",
+                      "category", "amt_nominal") %in% names(r)))
+    expect_identical(sort(unique(r$category)),
+                     c("Fund Balances", "Insurance Trust Balances"))
+    expect_false(is.null(attr(r, "provenance")))
+    expect_identical(attr(r, "provenance")$verb, "cog_balances")
+  })
+})
+
+test_that('category = "Fund Balances" is exactly the general family', {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", 2019, category = "Fund Balances")
+    expect_identical(unique(r$balance_subtype), "general")
+    codes <- sort(unlist(strsplit(paste(r$codes_included, collapse = ","), ",")))
+    expect_identical(codes, c("W01", "W31", "W61"))
+  })
+})
+
+test_that("no flow code can reach cog_balances", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", c(2011, 2012, 2019, 2020))
+    got <- unique(unlist(strsplit(paste(r$codes_included, collapse = ","), ",")))
+
+    # The expected set is read from the RAW corpus, never from the verb --
+    # verifying an absence through the filter that creates it proves nothing.
+    # A fresh, direct DuckDB connection against the raw parquet files (never
+    # cog_open()'s session, never balance_long/balance_annotated) reads
+    # parquet natively -- no arrow dependency needed (see CLAUDE.md).
+    con2 <- DBI::dbConnect(duckdb::duckdb())
+    on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
+    cats_path <- file.path(fixture_corpus_path(), "data", "summary_categories.parquet")
+    balance_codes <- DBI::dbGetQuery(con2, sprintf(
+      "SELECT item_code FROM read_parquet(%s) WHERE category_type = 'balance'",
+      uscogdata:::.sql_lit_chr(cats_path)
+    ))$item_code
+
+    expect_true(length(got) > 0L)
+    expect_true(all(got %in% balance_codes))
+  })
+})
+
+test_that("every balance_subtype maps to exactly one category", {
+  skip_if_no_corpus()
+  # Dropping the `subtype` argument is only safe while this tree holds. If the
+  # pipeline ever gives a balance subtype a second category, `category` becomes
+  # a lossy filter -- fail HERE rather than in a user's analysis. Read via a
+  # fresh direct DuckDB connection against the raw parquet file, not through
+  # any registered view.
+  con2 <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
+  cats_path <- file.path(fixture_corpus_path(), "data", "summary_categories.parquet")
+  b <- DBI::dbGetQuery(con2, sprintf(
+    "SELECT category, balance_subtype FROM read_parquet(%s) WHERE category_type = 'balance'",
+    uscogdata:::.sql_lit_chr(cats_path)
+  ))
+  per_subtype <- tapply(b$category, b$balance_subtype,
+                        function(x) length(unique(x)))
+  expect_true(all(per_subtype == 1L))
+})
+
+test_that("cog_balances records found + missing govids in provenance", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    suppressMessages(
+      r <- cog_balances(c("550000227544", "XXXINVALID"), 2019)
+    )
+    prov <- attr(r, "provenance")
+    expect_equal(sort(prov$scope$govids_found), "550000227544")
+    expect_equal(sort(prov$scope$govids_missing), "XXXINVALID")
+  })
+})
+
+test_that("per_capita divides holdings by population", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    plain <- cog_balances("550000227544", 2019, category = "Fund Balances")
+    pc    <- cog_balances("550000227544", 2019, category = "Fund Balances",
+                          per_capita = TRUE)
+    expect_true("amt_per_capita_nominal" %in% names(pc))
+    expect_true("pop_source" %in% names(pc))
+    expect_identical(pc$amt_nominal, plain$amt_nominal)
+
+    # Assert against the denominator read from the corpus, NOT against a
+    # quantity derived from amt_per_capita_nominal itself -- dividing the
+    # column back out would be tautological and would pass on any value.
+    pop <- DBI::dbGetQuery(cog_open(), sprintf(
+      "SELECT population FROM gov_population_yearly
+       WHERE canonical_govid = %s AND year = 2019",
+      uscogdata:::.sql_lit_chr("550000227544")
+    ))$population
+    expect_length(pop, 1L)
+    expect_equal(pc$amt_per_capita_nominal, pc$amt_nominal / pop,
+                 tolerance = 1e-8)
+
+    prov <- attr(pc, "provenance")
+    expect_true(prov$transformations$per_capita$applied)
+  })
+})
+
+test_that("adjust_to_year adds real dollars", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", 2012, category = "Fund Balances",
+                      adjust_to_year = 2020)
+    expect_true("amt_real" %in% names(r))
+    # 2012 dollars inflated to 2020 must exceed nominal.
+    expect_true(all(r$amt_real > r$amt_nominal))
+    prov <- attr(r, "provenance")
+    expect_true(prov$transformations$inflation$applied)
+    expect_identical(prov$transformations$inflation$base_year, 2020L)
+  })
+})
+
+test_that("per_capita and adjust_to_year compose", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", 2012, category = "Fund Balances",
+                      per_capita = TRUE, adjust_to_year = 2020)
+    expect_true("amt_per_capita_real" %in% names(r))
+    # The per-capita column must be deflated by the SAME factor as the level
+    # column -- this is what the ordering at R/balances.R:101-103 guarantees.
+    # .attach_real_dollars() silently no-ops on the per-capita leg when
+    # amt_per_capita_nominal does not exist yet (R/spending.R:664), so
+    # reversing those two calls drops this column with no error at all.
+    expect_equal(r$amt_per_capita_real / r$amt_per_capita_nominal,
+                 r$amt_real / r$amt_nominal, tolerance = 1e-8)
+
+    # And the documented condition is a conjunction: adjust_to_year ALONE
+    # must not produce amt_per_capita_real (pins the @return wording).
+    r2 <- cog_balances("550000227544", 2012, category = "Fund Balances",
+                       adjust_to_year = 2020)
+    expect_true("amt_real" %in% names(r2))
+    expect_false("amt_per_capita_real" %in% names(r2))
+  })
+})
+
+# --- input validation ------------------------------------------------------
+
+test_that("cog_balances validates its inputs like the money verbs", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    G <- "550000227544"
+    # Pinned to the message, not bare expect_error(): every one of these
+    # already produces *some* error or *some* quiet wrong answer today --
+    # years = integer(0) leaks `Parser Error ... AND year IN ()` with the
+    # generated SQL, recipe = c("a","b") throws "the condition has length > 1",
+    # and the govid/category cases return 0 rows with no error at all.
+    expect_error(cog_balances(G, integer(0)), "non-empty integer vector")
+    expect_error(cog_balances(character(0), 2019), "non-empty character vector")
+    expect_error(cog_balances(G, 2019, category = 5), "must be character or NULL")
+    expect_error(cog_balances(G, 2019, recipe = c("a", "b")),
+                 "length-1 character string")
+  })
+})
+
+test_that("validation runs after govid coercion, so a data frame still works", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # .validate_verb_inputs() asserts is.character(govid); it must therefore
+    # run AFTER .coerce_govid_input(), never before, or the documented
+    # data-frame input (cog_gov_search() output) would abort.
+    df <- data.frame(canonical_govid = "550000227544", stringsAsFactors = FALSE)
+    r <- suppressMessages(cog_balances(df, 2019))
+    expect_true(nrow(r) > 0L)
+    expect_identical(unique(r$canonical_govid), "550000227544")
+  })
+})
+
+test_that("recipe and category are mutually exclusive", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    expect_error(
+      cog_balances("550000227544", c(2011, 2012),
+                   category = "Fund Balances",
+                   recipe   = "cash_securities_z77_wide"),
+      class = "uscogdata_recipe_category_conflict"
+    )
+  })
+})
+
+# --- recipe = : the wide-era holdings bridge -------------------------------
+
+test_that("recipe bridges the wide era into the modern one", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", c(2011, 2012),
+                      recipe = "cash_securities_z77_wide")
+    # .run_recipe()'s SQL returns `long.year` as a DOUBLE (a corpus-wide trait,
+    # not specific to this recipe -- see the money-verb recipe tests, which
+    # only ever assert on it with expect_equal), so compare numerically rather
+    # than with expect_identical()'s type-strict comparison.
+    expect_equal(sort(r$year), c(2011, 2012))
+
+    # The 2011 leg can ONLY come from X40, which is 100% is_aggregate = TRUE
+    # and therefore invisible to balance_long. If the recipe path ever starts
+    # filtering aggregates, a 45-year series silently truncates to five --
+    # this is the regression guard for phase_r_harmonization_review.md § 0.2.
+    codes <- attr(r, "provenance")$codes_summed$observed
+    expect_true("X40" %in% codes)
+    expect_true("Z77" %in% codes)
+    expect_true(all(r$amt_nominal > 0))
+
+    prov <- attr(r, "provenance")
+    expect_identical(prov$recipe$recipe_id, "cash_securities_z77_wide")
+  })
+})
+
+test_that("the FY2002 book-to-market basis change is disclosed on the recipe path", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # 2002 is in the year vector deliberately, and must stay -- do not
+    # "simplify" this back to c(2011, 2012).
+    #
+    # .build_series_break_refs() (R/series_breaks.R, shared with every verb)
+    # matches breaks with `break_year BETWEEN min(years) AND max(years)`, and
+    # SB195's break_year is 2002. A c(2011, 2012) span never crosses the
+    # FY2002 book -> market change -- that whole span sits after it, on one
+    # consistent basis -- so NOT disclosing SB195 there is correct behaviour,
+    # not a gap (same reasoning as the "a request that never crosses the
+    # boundary is not affected by it" comment on .build_corpus_break_refs()).
+    #
+    # The property actually worth testing is: a recipe query that observes
+    # X40 AND spans FY2002 discloses SB195. This fixture has no 2002
+    # partition data for X40/Z77 (confirmed: only 2011/2012/2019/2020
+    # partitions exist), so including 2002 in `years` widens the
+    # break-matching window without changing which rows the recipe join
+    # returns -- verified empirically: r$year below is exactly {2011, 2012}
+    # whether or not 2002 is in the request (see task-4-report.md).
+    # Removing 2002 would silently turn this back into the non-crossing case
+    # above and destroy the test's purpose.
+    r <- cog_balances("550000227544", c(2002, 2011, 2012),
+                      recipe = "cash_securities_z77_wide")
+    expect_equal(sort(r$year), c(2011, 2012))
+    refs <- attr(r, "provenance")$series_break_refs
+    # SB195 sits on fin_code X40; it can only fire where X40 is observed,
+    # which is exactly the recipe path.
+    expect_true("SB195" %in% refs)
+  })
+})
+
+test_that("the second holdings bridge works too", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # X41 -> Z78, the securities counterpart. Wisconsin carries X41 in 2011
+    # and Z78 in 2012, so both legs are exercised.
+    r <- cog_balances("550000227544", c(2011, 2012),
+                      recipe = "cash_securities_z78_wide")
+    codes <- attr(r, "provenance")$codes_summed$observed
+    expect_true(all(c("X41", "Z78") %in% codes))
+    expect_equal(sort(r$year), c(2011, 2012))
+  })
+})
+
+test_that("an unknown recipe id is rejected", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # Asserted on the CLASS .validate_recipe_id() sets (R/recipes.R:84).
+    # Without it the test is non-discriminating: deleting the validation call
+    # leaves .recipe_components() returning 0 rows and comps$label[[1]]
+    # throwing "subscript out of bounds", which a bare expect_error() accepts
+    # while the user loses the curated "valid recipe ids are ..." message.
+    expect_error(
+      cog_balances("550000227544", 2019, recipe = "no_such_recipe"),
+      class = "uscogdata_unknown_recipe"
+    )
+  })
+})
+
+# --- balance_caveats: GAAP disclosure + measured coverage windows ----------
+
+test_that("balance_caveats is always present and flags the GAAP distinction", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", 2019)
+    cav <- attr(r, "provenance")$balance_caveats
+    expect_false(is.null(cav))
+    expect_true(cav$not_gaap)
+  })
+})
+
+test_that("coverage_window is computed from the corpus, not hardcoded", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_balances("550000227544", c(2011, 2012, 2019, 2020))
+    cav <- attr(r, "provenance")$balance_caveats
+
+    # Read the "general" family's true year extent independently, via a
+    # fresh DuckDB connection against the raw parquet files (never through
+    # balance_long/.balance_caveats() itself, and never via arrow -- this
+    # package reads parquet through DuckDB only, see CLAUDE.md). Replicates
+    # the same predicates 26-balance_long.sql applies (category_type =
+    # 'balance', NOT is_aggregate) so this is a faithful, independent
+    # measurement rather than a re-statement of the view under test.
+    con2 <- DBI::dbConnect(duckdb::duckdb())
+    on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
+    long_glob <- file.path(fixture_corpus_path(), "data", "long", "**", "*.parquet")
+    cats_path <- file.path(fixture_corpus_path(), "data", "summary_categories.parquet")
+    obs <- DBI::dbGetQuery(con2, sprintf(
+      "SELECT MIN(l.year) AS y0, MAX(l.year) AS y1
+       FROM read_parquet(%s, hive_partitioning = true) l
+       JOIN read_parquet(%s) c USING (item_code)
+       WHERE c.balance_subtype = 'general' AND NOT l.is_aggregate",
+      uscogdata:::.sql_lit_chr(long_glob), uscogdata:::.sql_lit_chr(cats_path)
+    ))
+
+    expect_identical(as.integer(cav$coverage_window$general),
+                     c(as.integer(obs$y0), as.integer(obs$y1)))
+  })
+})
+
+test_that("coverage_window covers every corpus subtype, not just observed ones", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # Deliberate contract (provenance-v1.json): the window block is corpus-
+    # scoped so a caller can ask "is there a family I missed?", while
+    # `truncated` is the observed-scoped field. A single-category query must
+    # therefore still report every balance family in the mounted corpus.
+    r <- cog_balances("550000227544", 2019, category = "Fund Balances")
+    expect_identical(unique(r$balance_subtype), "general")
+
+    con2 <- DBI::dbConnect(duckdb::duckdb())
+    on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
+    cats_path <- file.path(fixture_corpus_path(), "data", "summary_categories.parquet")
+    all_subtypes <- DBI::dbGetQuery(con2, sprintf(
+      "SELECT DISTINCT balance_subtype FROM read_parquet(%s)
+       WHERE balance_subtype IS NOT NULL",
+      uscogdata:::.sql_lit_chr(cats_path)
+    ))$balance_subtype
+
+    cav <- attr(r, "provenance")$balance_caveats
+    expect_setequal(names(cav$coverage_window), all_subtypes)
+    expect_true(length(all_subtypes) > 1L)
+    # ...while `truncated` stays scoped to what this query actually observed.
+    expect_true(all(cav$truncated %in% unique(r$balance_subtype)))
+  })
+})
+
+test_that("the corpus-constant coverage windows are memoised per session", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # The windows query has no govid/year predicate: its answer depends only
+    # on which corpus is mounted, so re-running the full balance_long scan on
+    # every call is pure waste (35% of verb runtime on the fixture). Same
+    # memoise-and-invalidate pattern as .uscogdata_env$manifest.
+    expect_null(uscogdata:::.uscogdata_env$balance_coverage_windows)
+    suppressMessages(cog_balances("550000227544", 2019))
+    memo <- uscogdata:::.uscogdata_env$balance_coverage_windows
+    expect_false(is.null(memo))
+    expect_true("general" %in% names(memo))
+
+    uscogdata:::cog_close()
+    expect_null(uscogdata:::.uscogdata_env$balance_coverage_windows)
+  })
+})
+
+test_that("a request past a family's coverage window is flagged", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # employee_retirement (X21/X30/X47/Z77/Z78) genuinely ends at FY2016 in
+    # the LIVE corpus -- Census moved employee retirement reporting to the
+    # Annual Survey of Public Pensions after that year. This bundled FIXTURE
+    # doesn't carry 2013-2016 at all (only 2011/2012/2019/2020 are present),
+    # so the family's *observed* max here is 2012, not 2016. Either way the
+    # requested span (2012, 2019) reaches past what the family covers in
+    # THIS corpus, which is what makes .balance_caveats() flag it -- the
+    # assertion below is about the fixture's measured window, not the FY2016
+    # live-corpus cutoff.
+    r <- cog_balances("550000227544", c(2012, 2019))
+    cav <- attr(r, "provenance")$balance_caveats
+    expect_true("employee_retirement" %in% cav$truncated)
+  })
+})
+
+test_that("the provenance schema documents balance_caveats", {
+  sch <- jsonlite::fromJSON(
+    system.file("schemas", "provenance-v1.json", package = "uscogdata"),
+    simplifyVector = FALSE
+  )
+  expect_true("balance_caveats" %in% names(sch$properties))
+})
+
+test_that("cog_explain surfaces the balance caveats", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # Asserted on the RENDERED text, not on prov$balance_caveats: the field
+    # is already covered above, and the once-per-session cli_inform() means
+    # cog_explain() is the only surface a caller who missed (or suppressed)
+    # the first message can still audit.
+    r <- suppressMessages(cog_balances("550000227544", c(2012, 2019)))
+    # Both streams: cli routes most of its output through conditions that
+    # land on stderr, so a stdout-only capture would be empty (the pattern
+    # used throughout test-explain.R).
+    out <- paste(c(capture.output(cog_explain(r)),
+                   capture.output(cog_explain(r), type = "message")),
+                 collapse = "\n")
+    expect_match(out, "GAAP")
+    expect_match(out, "employee_retirement")
+  })
+})
+
+test_that("cog_explain on a money-verb result has no balance caveat section", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- suppressMessages(cog_spending("550000227544", 2019))
+    out <- paste(c(capture.output(cog_explain(r)),
+                   capture.output(cog_explain(r), type = "message")),
+                 collapse = "\n")
+    # Guard against the capture itself being vacuous: the section must be
+    # absent from output that demonstrably contains the rest of the report.
+    expect_match(out, "Data vintage")
+    expect_false(grepl("GAAP", out))
+  })
+})
+
+test_that("the caveat message fires once per session", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    expect_message(cog_balances("550000227544", 2019), "not.*GAAP")
+    expect_no_message(cog_balances("550000227544", 2020))
+  })
+})

tests/testthat/test-categories.R

@@ -8,22 +8,55 @@ test_that("cog_categories returns all categories grouped by subtype", {
   expect_gt(nrow(r), 10L)
   # corpus preserves Census-native "expenditure" vocabulary; the API takes
   # "spending" as a friendlier alias.
-  expect_setequal(unique(r$category_type), c("expenditure", "revenue"))
+  #
+  # `balance` joined as a third category_type with the cash-and-security
+  # holding codes (pipeline#76). `cog_categories()` is a CATALOGUE verb, not a
+  # money verb, so it surfaces every category_type the corpus carries -- the
+  # stock/flow guard belongs on cog_spending()/cog_revenue(), which must never
+  # return a balance row.
+  expect_setequal(unique(r$category_type),
+                  c("expenditure", "revenue", "balance"))
 })
 
 test_that("cog_categories(type = 'spending') returns only expenditure rows", {
   skip_if_no_corpus()
   r <- cog_categories(type = "spending")
   expect_true(all(r$category_type == "expenditure"))
-  expect_true(all(r$subtype %in% c("operations", "capital")))
+  # "assistance" (the J-prefix aid/benefit codes) joined the vocabulary with
+  # the crosswalk completion in cog_pipeline#60/#65 -- every flow code
+  # carrying dollars now maps to a category.
+  # `interest` (I89, I91-I94) and `insurance_benefits` (Y05/Y06/Y14/Y53)
+  # joined with the I/Q/Y flow batch -- the last two characters of Census's
+  # expenditure taxonomy. `interest` is what makes the three-concept model
+  # computable: primary = direct minus debt service.
+  expect_true(all(r$subtype %in%
+                  c("operations", "capital", "intergovernmental", "assistance",
+                    "interest", "insurance_benefits")))
+})
+
+test_that("cog_categories surfaces the intergovernmental spending subtype", {
+  skip_if_no_corpus()
+  r <- cog_categories(type = "spending")
+  expect_true("intergovernmental" %in% r$subtype)
+  # IG rows reuse the existing functional categories -- they add a subtype,
+  # not new category values.
+  ig_cats     <- sort(unique(r$category[r$subtype == "intergovernmental"]))
+  direct_cats <- sort(unique(r$category[r$subtype != "intergovernmental"]))
+  expect_true(all(ig_cats %in% c(direct_cats, "Other Education")))
 })
 
 test_that("cog_categories(type = 'revenue') returns only revenue rows", {
   skip_if_no_corpus()
   r <- cog_categories(type = "revenue")
   expect_true(all(r$category_type == "revenue"))
+  # The four non-general subtypes are deliberately NOT own_source: Census's
+  # General Revenue excludes insurance trust (Y01 alone is $1.31T corpus-wide,
+  # plus the employee-retirement X codes), utility (A91-A94) and liquor store
+  # (A90) revenue by definition, which is what makes both of its published
+  # revenue concepts computable -- see `revenue_concept` in `?cog_revenue`.
   expect_true(all(r$subtype %in%
-                  c("own_source", "federal", "state", "local_aid")))
+                  c("own_source", "federal", "state", "local_aid",
+                    "insurance_trust", "utility", "liquor_store")))
 })
 
 test_that("cog_categories(pattern = ...) filters case-insensitively", {
@@ -60,3 +93,40 @@ test_that("cog_categories sorted by category_type, category, subtype", {
 test_that("cog_categories rejects invalid type", {
   expect_error(cog_categories(type = "both"), "type")
 })
+
+test_that("cog_categories() surfaces balance subtypes", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    cc <- cog_categories()
+    b <- cc[cc$category_type == "balance", ]
+    expect_true(nrow(b) > 0L)
+
+    # Every balance row must carry its subtype. Before the COALESCE included
+    # balance_subtype these were all NA, which silently made the balance
+    # taxonomy undiscoverable -- cog-api derives its subtype vocabulary from
+    # this function, so an NA here becomes an unusable API parameter.
+    expect_false(any(is.na(b$subtype)))
+
+    # The exact set, read independently from the crosswalk rather than from
+    # the function under test.
+    con2 <- DBI::dbConnect(duckdb::duckdb())
+    on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)
+    p <- file.path(fixture_corpus_path(), "data", "summary_categories.parquet")
+    want <- DBI::dbGetQuery(con2, sprintf(
+      "SELECT DISTINCT balance_subtype FROM read_parquet(%s)
+       WHERE category_type = 'balance' AND balance_subtype IS NOT NULL
+       ORDER BY 1", uscogdata:::.sql_lit_chr(p)))$balance_subtype
+    expect_true(length(want) > 1L)
+    expect_identical(sort(unique(b$subtype)), sort(want))
+  })
+})
+
+test_that('cog_categories(type = "balance") filters to holdings', {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    b <- cog_categories(type = "balance")
+    expect_true(nrow(b) > 0L)
+    expect_identical(unique(b$category_type), "balance")
+    expect_false(any(is.na(b$subtype)))
+  })
+})

tests/testthat/test-complete.R

@@ -0,0 +1,193 @@
+# tests/testthat/test-complete.R
+#
+# uscogdata#18. The published corpus no longer stores the wide era's explicit
+# zeros (cog_pipeline#64, series break SB194), so absence means two different
+# things:
+#
+#   <= FY2011 (dense_source) : cell absent => Census published $0
+#   >= FY2012 (sparse_source): cell absent => not reported, unknown
+#
+# `complete = TRUE` fills the requested grid from `code_set` and stamps every
+# row's `value_source` so the two are distinguishable. Expected row sets here
+# are built from the corpus parquet directly, never from the verb under test --
+# verifying what a filter does through that same filter proves nothing.
+
+# The (subtype, category) cells that SHOULD exist for one government-year:
+# every code in force for that government's type, mapped through
+# summary_categories, matching the verb's crosswalk subtype scope (the
+# default concept, `primary`, is operations/capital/assistance -- see
+# uscogdata#11) and excluding aggregate-flagged codes (which
+# spending_long/revenue_long drop).
+raw_expected_cells <- function(govid, year, subtypes, subtype_col) {
+  fx <- sub("/$", "", Sys.getenv("USCOGDATA_URL"))
+  q <- function(f) sprintf("read_parquet('%s/data/%s')", fx, f)
+  wt_raw_query(sprintf(
+    "SELECT DISTINCT c.%s AS subtype, c.category
+     FROM %s cs
+     JOIN %s x ON x.govs_type = cs.type
+     JOIN %s c ON c.item_code = cs.item_code
+     WHERE x.canonical_govid = '%s'
+       AND cs.year = %d
+       AND NOT cs.is_aggregate
+       AND c.category IS NOT NULL
+       AND c.%s IN (%s)",
+    subtype_col, q("code_set.parquet"), q("canonical_fips_xwalk.parquet"),
+    q("summary_categories.parquet"), govid, year,
+    subtype_col, paste0("'", subtypes, "'", collapse = ",")
+  ))
+}
+
+# The default expenditure concept's subtype scope, mirrored from
+# R/spending.R's .spend_subtypes_primary.
+primary_subtypes <- c("operations", "capital", "assistance")
+
+test_that("complete = FALSE is the default and changes nothing", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    plain <- cog_spending("121011212191", 2011L)
+    explicit <- cog_spending("121011212191", 2011L, complete = FALSE)
+    expect_equal(nrow(plain), nrow(explicit))
+    expect_false("value_source" %in% names(plain))
+  })
+})
+
+test_that("complete = TRUE round-trips a dense-source year to the pre-sparsification cells", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # FY2011 is dense_source: before sparsification this government carried a
+    # row for every code in force, most of them $0. complete = TRUE must
+    # reproduce that cell set exactly.
+    r <- cog_spending("121011212191", 2011L, complete = TRUE)
+    expected <- raw_expected_cells("121011212191", 2011L,
+                                   primary_subtypes, "spend_subtype")
+
+    key <- function(sub, cat) paste(sub, cat, sep = "|")
+    expect_setequal(key(r$spend_subtype, r$category),
+                    key(expected$subtype, expected$category))
+    expect_gt(nrow(expected), 0L)
+
+    # Every filled cell in a dense-source year is a Census-published $0 --
+    # never "unknown", which is what the modern era's absences mean.
+    expect_setequal(unique(r$value_source), c("reported", "census_zero"))
+    expect_true(all(r$amt_nominal[r$value_source == "census_zero"] == 0))
+    expect_true(all(r$amt_nominal[r$value_source == "reported"] != 0))
+  })
+})
+
+test_that("complete = TRUE preserves the reported rows and their amounts exactly", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    plain <- cog_spending("121011212191", 2011L)
+    full  <- cog_spending("121011212191", 2011L, complete = TRUE)
+
+    # Filling adds rows; it must never alter or drop one.
+    expect_gt(nrow(full), nrow(plain))
+    reported <- full[full$value_source == "reported", ]
+    expect_equal(nrow(reported), nrow(plain))
+    expect_equal(sum(reported$amt_nominal), sum(plain$amt_nominal))
+    # ... and the total is unchanged, because every added cell is $0.
+    expect_equal(sum(full$amt_nominal, na.rm = TRUE), sum(plain$amt_nominal))
+  })
+})
+
+test_that("a sparse-source year's absences are unknown, not zero", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # FY2019 is sparse_source: an absent cell means the government did not
+    # report, which is NOT a zero. Filling those with 0 would invent data --
+    # the exact error the representation contract exists to prevent.
+    r <- cog_spending("121011212191", 2019L, complete = TRUE)
+    filled <- r[r$value_source != "reported", ]
+    expect_gt(nrow(filled), 0L)
+    expect_true(all(filled$value_source == "not_reported"))
+    expect_true(all(is.na(filled$amt_nominal)))
+    expect_false(any(r$value_source == "census_zero"))
+  })
+})
+
+test_that("the fill is scoped to each government's own type", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # Filling against the union of all types would invent cells for codes a
+    # county can never report. Every filled category must be one that
+    # code_set puts in force for type 1 (county) specifically.
+    r <- cog_spending("121011212191", 2011L, complete = TRUE)
+    county_cells <- raw_expected_cells("121011212191", 2011L,
+                                       primary_subtypes, "spend_subtype")
+    expect_true(all(r$category %in% county_cells$category))
+  })
+})
+
+test_that("complete = TRUE respects the category filter", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_spending("121011212191", 2011L, category = "Police",
+                      complete = TRUE)
+    expect_true(all(r$category == "Police"))
+    expect_true("value_source" %in% names(r))
+  })
+})
+
+test_that("cog_revenue() completes on its own flow", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_revenue("121011212191", 2011L, complete = TRUE)
+    expected <- raw_expected_cells("121011212191", 2011L,
+                                   c("own_source", "federal", "state", "local_aid"),
+                                   "revenue_subtype")
+    key <- function(sub, cat) paste(sub, cat, sep = "|")
+    expect_setequal(key(r$revenue_subtype, r$category),
+                    key(expected$subtype, expected$category))
+    expect_setequal(unique(r$value_source), c("reported", "census_zero"))
+  })
+})
+
+test_that("provenance records the completion and its absence rule", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    prov <- attr(cog_spending("121011212191", 2011L, complete = TRUE),
+                 "provenance")
+    expect_true(prov$completion$applied)
+    expect_equal(prov$completion$absence_means$`2011`, "census_zero")
+    expect_gt(prov$completion$rows_filled, 0L)
+
+    off <- attr(cog_spending("121011212191", 2011L), "provenance")
+    expect_false(off$completion$applied)
+    expect_equal(off$completion$rows_filled, 0L)
+  })
+})
+
+test_that("complete = TRUE is refused where the fill would be guesswork", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # A recipe defines its own component codes and does not go through
+    # summary_categories at all, so there is no grid to fill from.
+    expect_error(
+      cog_spending("121011212191", 2011L, recipe = "corrections_combined",
+                   complete = TRUE),
+      class = "uscogdata_complete_unsupported"
+    )
+    # The intergovernmental leg keeps aggregate rows by design
+    # (inst/sql/24-ig_long.sql), so its grid is not code_set's grid.
+    expect_error(
+      cog_spending("121011212191", 2011L, expenditure_concept = "total",
+                   complete = TRUE),
+      class = "uscogdata_complete_unsupported"
+    )
+  })
+})
+
+test_that("complete = TRUE aborts on a corpus with no representation contract", {
+  skip_if_no_corpus()
+  # A corpus published before sparsification carries neither table, so there
+  # is nothing to fill from and no rule saying what an absence means. That
+  # must abort rather than guess.
+  with_corpus_missing_representation({
+    expect_error(
+      cog_spending("121011212191", 2011L, complete = TRUE),
+      class = "uscogdata_representation_unavailable"
+    )
+    # ... while an ordinary query on the same corpus still works.
+    expect_gt(nrow(cog_spending("121011212191", 2011L)), 0L)
+  })
+})

tests/testthat/test-config.R

@@ -26,3 +26,41 @@ test_that(".resolve_cache_dir falls back to R_user_dir", {
     })
   })
 })
+
+# ---------------------------------------------------------------------------
+# Trailing-slash normalization (uscogdata #3 follow-up).
+#
+# EVERY consumer builds paths by concatenation: paste0(url, "manifest.json")
+# (manifest.R), paste0(url, e$path) (mirror.R), and the parquet glob in
+# views.R. mirror.R:104 even comments 'url ends in "/"' -- an assumption the
+# package documents and relies on but never enforced.
+#
+# A URL missing its trailing slash therefore fails SILENTLY and confusingly:
+#   HTTPS -> ".../downloadmanifest.json" -> the host answers with an HTML 404
+#            page -> the jsonlite lexical error that issue #3 reported;
+#   local -> ".../corpusdata/long/**/*.parquet" -> DuckDB "No files found".
+# Neither message points at the real cause. Normalize once, at resolution.
+# ---------------------------------------------------------------------------
+
+test_that(".resolve_url appends a missing trailing slash", {
+  withr::local_envvar(USCOGDATA_URL = "https://example.org/s/TOKEN/download")
+  expect_equal(.resolve_url(), "https://example.org/s/TOKEN/download/")
+})
+
+test_that(".resolve_url leaves an existing trailing slash alone", {
+  withr::local_envvar(USCOGDATA_URL = "https://example.org/s/TOKEN/download/")
+  expect_equal(.resolve_url(), "https://example.org/s/TOKEN/download/")
+})
+
+test_that(".resolve_url normalizes a local path without a trailing slash", {
+  withr::local_envvar(USCOGDATA_URL = "/tmp/corpus")
+  expect_equal(.resolve_url(), "/tmp/corpus/")
+})
+
+test_that(".resolve_url does not invent a slash for an empty setting", {
+  # An unset/empty URL must stay empty so the "not configured" guard in
+  # manifest.R still fires, rather than degrading into a bare "/" root.
+  withr::local_envvar(USCOGDATA_URL = "")
+  withr::local_options(uscogdata.url = "")
+  expect_equal(.resolve_url(), "")
+})

tests/testthat/test-corpus-breaks.R

@@ -0,0 +1,94 @@
+# tests/testthat/test-corpus-breaks.R
+#
+# uscogdata#19. Four catalogued series breaks carry fin_code = "ALL" -- they
+# are caveats about the corpus itself rather than about one item code:
+#
+#   SB085  1977  dollar precision across the 1976/1977 boundary
+#   SB087  2002  imputation exclusion FY2002-2006
+#   SB194  2012  dense -> sparse representation change
+#   SB086  2017  government ID scheme change
+#
+# .build_series_break_refs() matches `fin_code IN (<codes in the result>)`,
+# and no row's item_code is ever the literal "ALL", so none of them could
+# ever reach a user. They now travel in their own provenance field,
+# `corpus_break_refs`, which keeps them distinguishable from the
+# code-specific `series_break_refs` (an ALL caveat qualifies the whole
+# result, not one series).
+
+test_that("corpus_break_refs surfaces an ALL-scoped break the year range spans", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # SB194 sits at FY2012 -- the dense/sparse boundary. A query spanning
+    # 2011 -> 2012 straddles it, and this is the case cog_pipeline#64's
+    # DoD 4 intended to reach users.
+    r <- cog_spending("121011212191", 2011:2012, "Police")
+    prov <- attr(r, "provenance")
+    expect_true("SB194" %in% prov$corpus_break_refs)
+  })
+})
+
+test_that("corpus_break_refs stays empty when no ALL break falls in the range", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    # 2019-2020 spans no catalogued corpus-wide break.
+    r <- cog_spending("121011212191", 2019:2020, "Police")
+    expect_equal(attr(r, "provenance")$corpus_break_refs, character(0))
+  })
+})
+
+test_that("corpus_break_refs and series_break_refs stay disjoint", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_spending("121011212191", 2011:2012, "Police")
+    prov <- attr(r, "provenance")
+    expect_type(prov$series_break_refs, "character")
+    expect_type(prov$corpus_break_refs, "character")
+    # An ALL caveat must never masquerade as a break in a specific series.
+    expect_length(intersect(prov$series_break_refs, prov$corpus_break_refs), 0L)
+    expect_false("SB194" %in% prov$series_break_refs)
+  })
+})
+
+test_that(".build_corpus_break_refs matches on the break_year window alone", {
+  skip_if_no_corpus()
+  con <- cog_open()
+  on.exit(cog_close())
+
+  # SB085's boundary is 1976/1977, outside the fixture's partitions -- the
+  # series_breaks table is a full cross-vintage registry, so the matching
+  # logic is testable there even though no long partition covers it.
+  expect_true("SB085" %in% uscogdata:::.build_corpus_break_refs(
+    con, years = 1975:1980, schema_version = 6L
+  ))
+  # ... and does not fire for a range that misses it, unlike a filter keyed
+  # on the era rather than the boundary.
+  expect_false("SB085" %in% uscogdata:::.build_corpus_break_refs(
+    con, years = 1978:1980, schema_version = 6L
+  ))
+
+  # Unlike code-specific refs, these do not depend on which codes a result
+  # happens to contain -- that dependency is the whole defect.
+  expect_setequal(
+    uscogdata:::.build_corpus_break_refs(con, years = 2001:2003, schema_version = 6L),
+    "SB087"
+  )
+
+  # Gated on schema_version >= 5: series_breaks_pq is not registered below it.
+  expect_equal(
+    uscogdata:::.build_corpus_break_refs(con, years = 2011:2012, schema_version = 4L),
+    character(0)
+  )
+})
+
+test_that("cog_explain() prints corpus-wide caveats under their own heading", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_spending("121011212191", 2011:2012, "Police")
+    out <- paste(c(
+      capture.output(cog_explain(r)),
+      capture.output(cog_explain(r), type = "message")
+    ), collapse = "\n")
+    expect_match(out, "Corpus-wide caveats", fixed = TRUE)
+    expect_match(out, "SB194", fixed = TRUE)
+  })
+})

tests/testthat/test-coverage-disclosure.R

@@ -0,0 +1,103 @@
+# Madison walkthrough audit -- findings F-020 and F-023. Tracked as uscogdata#13.
+# See docs/walkthroughs/FINDINGS.md in cog_explorer.
+#
+# The owner's settled design (2026-07-28): a `coverage` argument on
+# cog_geographic_rollup(), cog_find_peers()/cog_peer_compare() and their
+# cog-api equivalents --
+#   "all"        every unit that reported that year (today's behaviour, DEFAULT)
+#   "census"     census years only (years ending 2 or 7)
+#   "consistent" only units reporting in every requested year (balanced panel)
+# -- PLUS always-on coverage metadata on every result regardless of mode:
+# n_units_reporting, n_units_expected, is_census_year.
+#
+# Motivating principle: using these verbs correctly must not require the user to
+# know that the Census of Governments is a complete census only in years ending
+# in 2 and 7.
+#
+# The helper below accepts that metadata either as columns on the returned
+# tibble or as a per-year table in provenance$coverage -- the design fixes the
+# three field names and that they reach the caller, not the container.
+
+wt_coverage <- function(x) {
+  prov <- attr(x, "provenance")
+  cov <- prov$coverage
+  if (is.null(cov)) {
+    needed <- c("year", "n_units_reporting", "n_units_expected", "is_census_year")
+    expect_true(all(needed %in% names(x)))
+    cov <- unique(x[, needed])
+  }
+  cov[order(cov$year), ]
+}
+
+test_that("multi-government aggregates disclose reporting coverage on every result", {
+
+  # -- F-020: geographic rollups -------------------------------------------
+  # Wisconsin's city/village universe is 608 governments. On the bundled
+  # fixture, FY2012 (a census year) has 597 of them reporting while FY2019 and
+  # FY2020 (sample years) have 112 and 114 -- an 18%-98% swing that today's
+  # return value says nothing about. Counts cross-checked against the raw
+  # corpus, not through cog_geographic_rollup(), which is under test.
+  wi <- cog_gov_search(name = NULL, state = "WI", type = "city")
+  expect_equal(nrow(wi), 608L)
+
+  roll <- cog_geographic_rollup(govids = list(city = wi$canonical_govid),
+                                category = NULL, years = c(2011L, 2012L, 2019L, 2020L))
+  cov <- wt_coverage(roll)
+
+  expect_equal(cov$n_units_expected, rep(608L, 4L))
+  expect_equal(cov$n_units_reporting, c(152L, 597L, 112L, 114L))
+  expect_equal(cov$is_census_year, c(FALSE, TRUE, FALSE, FALSE))
+
+  # Cross-check against the raw partitions, scoped to the SAME universe the
+  # rollup was given -- the 608 govids above. Scoping instead on the long
+  # table's own `type`/`fips_state` asks a different question and answers 595:
+  # VERNON VILLAGE and WAUKESHA VILLAGE carry type = 3 there (their as-of-year
+  # identity, when they were townships) while the xwalk lists them as
+  # govs_type = 2 (their present identity, as villages). Schema v6 made the
+  # long table's geography present-harmonized and moved as-of-year to the
+  # *_asof columns, but `type` still reads as-of-year -- see .validate_schema()
+  # in R/manifest.R. n_units_reporting counts against the requested universe,
+  # so 597 is the number that answers "how many of the governments I asked
+  # about reported".
+  raw_2012 <- wt_raw_query(paste0(
+    "SELECT COUNT(DISTINCT canonical_govid) n FROM read_parquet('", wt_corpus_glob(), "') ",
+    "WHERE year = 2012 AND LEFT(item_code, 1) IN ('E','F','G') AND NOT is_aggregate ",
+    "AND canonical_govid IN (",
+    paste0("'", wi$canonical_govid, "'", collapse = ","), ")"))
+  expect_equal(cov$n_units_reporting[cov$year == 2012], as.integer(raw_2012$n[[1]]))
+
+  # -- F-023: peer cohorts --------------------------------------------------
+  # CHILTON CITY, WI (ACS population 4,017): a 15-peer cohort fixed at FY2012
+  # reports 15 of 15 in FY2012 and only 3 of 15 in FY2019 and FY2020. Nothing
+  # in cog_peer_compare()'s return distinguishes those years today.
+  chilton <- "552015177095"
+  peers <- cog_find_peers(chilton, year = 2012L, max_peers = 15L)
+  expect_equal(nrow(peers), 15L)
+
+  cmp <- cog_peer_compare(target_govid = chilton, peers = peers, category = NULL,
+                          years = c(2012L, 2019L, 2020L), per_capita = TRUE)
+  cov_peers <- wt_coverage(cmp)
+  expect_equal(cov_peers$n_units_expected, rep(15L, 3L))
+  expect_equal(cov_peers$n_units_reporting, c(15L, 3L, 3L))
+  expect_equal(cov_peers$is_census_year, c(TRUE, FALSE, FALSE))
+
+  # -- the three coverage modes --------------------------------------------
+  expect_equal(attr(cog_peer_compare(target_govid = chilton, peers = peers,
+                                     category = NULL, years = c(2012L, 2019L, 2020L),
+                                     per_capita = TRUE),
+                    "provenance")$coverage_mode, "all")     # unchanged default
+
+  consistent <- cog_peer_compare(target_govid = chilton, peers = peers,
+                                 category = NULL, years = c(2012L, 2019L, 2020L),
+                                 per_capita = TRUE, coverage = "consistent")
+  n_by_year <- tapply(consistent$canonical_govid[consistent$role == "peer"],
+                      consistent$year[consistent$role == "peer"],
+                      function(g) length(unique(g)))
+  expect_equal(unname(as.integer(n_by_year)), c(3L, 3L, 3L))  # balanced panel
+
+  census_only <- cog_geographic_rollup(govids = list(city = wi$canonical_govid),
+                                       category = NULL,
+                                       years = c(2011L, 2012L, 2019L, 2020L),
+                                       coverage = "census")
+  expect_equal(sort(unique(census_only$year)), 2012)
+})

tests/testthat/test-expenditure-concept.R

@@ -0,0 +1,553 @@
+test_that("the corpus contains no K-prefix rows, so the Direct leg omits K", {
+  con <- .ensure_session()
+  n <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS n FROM long WHERE LEFT(item_code, 1) = 'K'")$n
+  expect_equal(n, 0)
+
+  sql_files <- c("20-spending_long.sql", "22-spending_long_harmonized.sql")
+  for (f in sql_files) {
+    txt <- paste(readLines(system.file("sql", f, package = "uscogdata")),
+                 collapse = " ")
+    expect_false(grepl("'K'", txt, fixed = TRUE),
+                 label = paste(f, "must not reference the inert K prefix"))
+  }
+})
+
+test_that("expenditure_concept defaults to primary; direct matches it on a pure operations/capital category", {
+  gov <- "010000226085"                      # Alabama state government
+  base <- cog_spending(gov, years = 2019, category = "Police")
+  expect_equal(attr(base, "provenance")$expenditure_concept, "primary")
+  # Police maps only to operations/capital codes (E62/F62/G62), so the
+  # direct concept's extra subtypes (interest, insurance_benefits) cannot
+  # contribute and the two concepts must agree exactly here.
+  expl <- cog_spending(gov, years = 2019, category = "Police",
+                       expenditure_concept = "direct")
+  expect_equal(base$amt_nominal, expl$amt_nominal)
+  expect_false("intergovernmental" %in% base$spend_subtype)
+})
+
+test_that("expenditure_concept = 'total' adds an intergovernmental subtype", {
+  gov <- "010000226085"
+  d <- cog_spending(gov, years = 2019, category = "Police",
+                    expenditure_concept = "direct")
+  t <- cog_spending(gov, years = 2019, category = "Police",
+                    expenditure_concept = "total")
+  expect_true("intergovernmental" %in% t$spend_subtype)
+  # Direct rows are untouched; Total only ever ADDS. Use %in% rather than
+  # != : a category = NULL result can contain a NULL-subtype group (codes
+  # with no summary_categories row, e.g. E16/E21/E85/F16/F85/G16/G21/G85),
+  # and `NA != "intergovernmental"` is NA, not TRUE, which would silently
+  # smuggle an all-NA phantom row into dt.
+  dt <- t[!(t$spend_subtype %in% "intergovernmental"), ]
+  expect_equal(sort(dt$amt_nominal), sort(d$amt_nominal))
+  expect_gt(sum(t$amt_nominal), sum(d$amt_nominal))
+})
+
+test_that("legacy-era Total does not collapse to Direct (the is_aggregate trap)", {
+  # In the wide era the IG dollars live almost entirely on aggregate-flagged
+  # rows. A Total leg that inherited the Direct leg's NOT is_aggregate filter
+  # would silently return Total == Direct here.
+  gov <- "010000226085"
+  d <- cog_spending(gov, years = 2011, category = "Education K-12",
+                    expenditure_concept = "direct")
+  t <- cog_spending(gov, years = 2011, category = "Education K-12",
+                    expenditure_concept = "total")
+  expect_true("intergovernmental" %in% t$spend_subtype)
+  ig <- sum(t$amt_nominal[t$spend_subtype == "intergovernmental"])
+  expect_gt(ig, 0)
+  expect_gt(sum(t$amt_nominal), sum(d$amt_nominal))
+})
+
+test_that("the IG leg never includes the L-- family total", {
+  con <- .ensure_session()
+  codes <- DBI::dbGetQuery(con,
+    "SELECT DISTINCT item_code FROM ig_long")$item_code
+  expect_false(any(grepl("--$", codes)))
+  # Q joined the IG family with the crosswalk-membership rewrite
+  # (uscogdata#11 / F-017: Q11/Q12/Q18 are state payments to school systems).
+  expect_true(all(substr(codes, 1, 1) %in% c("M", "L", "Q")))
+})
+
+test_that("expenditure_concept rejects unknown values", {
+  expect_error(
+    cog_spending("010000226085", years = 2019, expenditure_concept = "gross"),
+    class = "rlang_error"
+  )
+})
+
+test_that("total composes with basis = 'raw' and basis = 'harmonized'", {
+  gov <- "010000226085"
+  h <- cog_spending(gov, years = 2011, category = "Education K-12",
+                    expenditure_concept = "total", basis = "harmonized")
+  r <- cog_spending(gov, years = 2011, category = "Education K-12",
+                    expenditure_concept = "total", basis = "raw")
+  ig_h <- sum(h$amt_nominal[h$spend_subtype == "intergovernmental"])
+  ig_r <- sum(r$amt_nominal[r$spend_subtype == "intergovernmental"])
+  # The only IG harmonization rule is M38 -> M36 (year-disjoint), so the IG
+  # total must agree between bases even though the code labels may differ.
+  expect_equal(ig_h, ig_r)
+})
+
+test_that("recipe = and expenditure_concept = 'total' together aborts", {
+  expect_error(
+    cog_spending("121011212191", 2020L, recipe = "corrections_combined",
+                expenditure_concept = "total"),
+    class = "uscogdata_recipe_concept_conflict"
+  )
+})
+
+test_that("aggregate-sourced IG dollars are flagged aggregate_fallback = TRUE (bool_or, not bool_and)", {
+  # Regression test: .build_verb_sql() originally used bool_and(is_aggregate)
+  # for aggregate_fallback, which is correct for the Direct leg (a group can
+  # never mix aggregate and non-aggregate rows there -- spending_long filters
+  # NOT is_aggregate) but wrong for the IG leg. The wide era is dense -- every
+  # government has a $0 row for every code in a family -- so a $0 leaf sits in
+  # the same (year, gov, subtype, category) group as the real aggregate row
+  # and flips bool_and() to FALSE. Measured: AL state 2011 had $5,740,775,000
+  # of aggregate-sourced IG dollars (Corrections $31,358,000 + Education K-12
+  # $5,152,385,000 + General Government $557,032,000) reporting
+  # aggregate_fallback = FALSE under bool_and(), with the only TRUE row being
+  # Transit Utilities at $0. bool_or() reports all of them correctly.
+  gov <- "010000226085"
+  t <- cog_spending(gov, years = 2011, category = "Education K-12",
+                    expenditure_concept = "total")
+  ig <- t[t$spend_subtype == "intergovernmental", ]
+  expect_equal(nrow(ig), 1L)
+  expect_true(ig$aggregate_fallback)
+  expect_true(nzchar(ig$notes))
+  expect_match(ig$notes, "Aggregate fallback applied", fixed = TRUE)
+})
+
+test_that("legacy aggregate IG codes are year-disjoint from their modern leaf components", {
+  # The safety of ig_long's deliberate omission of `NOT is_aggregate` (see
+  # inst/sql/24-ig_long.sql) rests entirely on each legacy code's AGGREGATE
+  # instance being year-disjoint from the modern leaf codes it rolls up --
+  # if a future corpus rebuild ever back-filled a leaf into a year where the
+  # code is still flagged aggregate, `total` would silently double-count and
+  # this suite would still pass. This test fails loudly if that ever
+  # happens.
+  #
+  # Note the invariant is scoped to the AGGREGATE flag, not bare code
+  # presence: M89/L89 do NOT disappear after the wide era the way M47/L47
+  # do -- they continue past 2011 as their OWN independent leaf line item
+  # (is_aggregate = FALSE) alongside M91-93/L91-93, which is fine because a
+  # non-aggregate M89/L89 no longer represents a rollup of those codes.
+  # (Verified in the fixture: M89/L89 are is_aggregate = TRUE only in 2011,
+  # when M91-93/L91-93 don't exist yet; from 2012 on M89/L89 are
+  # is_aggregate = FALSE leaves coexisting with M91-93/L91-93.)
+  #
+  # Pairs are the M/L-prefixed components (this package's ig_long only
+  # covers M/L; other prefixes in the same rollup, e.g. N/O/P/Q/R, fall
+  # outside its domain and are irrelevant here) enumerated in
+  # cog_pipeline's data/wide_to_long_xwalk.csv `full_desc` column (read
+  # once at authoring time, not at test time -- this test stays offline):
+  #   M47 "To local governments, total (includes N47, O47, P47, R47, and M94)"
+  #   M89 "To local governments, total (incl N89, O89, P89, R89, M91, M92, and M93)"
+  #   L47 "To state government (includes L94)"
+  #   L89 "To state government (includes L91, L92, and L93)"
+  con <- .ensure_session()
+  pairs <- list(
+    list(aggregate = "M47", components = "M94"),
+    list(aggregate = "M89", components = c("M91", "M92", "M93")),
+    list(aggregate = "L47", components = "L94"),
+    list(aggregate = "L89", components = c("L91", "L92", "L93"))
+  )
+  agg_years_by_code <- DBI::dbGetQuery(con,
+    "SELECT DISTINCT year, item_code FROM ig_long WHERE is_aggregate")
+  codes_by_year <- DBI::dbGetQuery(con, "SELECT DISTINCT year, item_code FROM ig_long")
+
+  for (p in pairs) {
+    agg_years <- agg_years_by_code$year[agg_years_by_code$item_code == p$aggregate]
+    for (yr in agg_years) {
+      codes_yr <- codes_by_year$item_code[codes_by_year$year == yr]
+      has_component <- any(p$components %in% codes_yr)
+      expect_false(
+        has_component,
+        label = sprintf(
+          "year %s has aggregate-flagged %s co-occurring with a modern component (%s)",
+          yr, p$aggregate, paste(p$components, collapse = ",")
+        )
+      )
+    }
+  }
+})
+
+test_that(".verb_spendrev rejects expenditure_concept = 'total' for a non-spending view_base", {
+  # cog_revenue() never exposes expenditure_concept and always resolves it
+  # to the "direct" default, so there is no revenue codepath that reaches
+  # this today -- but .verb_spendrev() is shared, and nothing else stops a
+  # future caller from passing expenditure_concept = "total" alongside
+  # view_base = "revenue_annotated", which would UNION expenditure M/L rows
+  # into a revenue result. Exercise the internal helper directly.
+  expect_error(
+    uscogdata:::.verb_spendrev(
+      verb = "cog_revenue_test", view_base = "revenue_annotated",
+      subtype_col = "revenue_subtype",
+      flow_prefixes = c("T", "A", "U", "B", "C", "D"),
+      call = quote(cog_revenue_test()),
+      govid = "010000226085", years = 2019L, category = NULL,
+      per_capita = FALSE, adjust_to_year = NULL, basis = "raw",
+      recipe = NULL, expenditure_concept = "total"
+    ),
+    class = "uscogdata_expenditure_concept_unsupported"
+  )
+})
+
+test_that("cog_geographic_rollup refuses expenditure_concept = 'total'", {
+  expect_error(
+    cog_geographic_rollup(
+      govids = list(state = "010000226085"),
+      category = "Police", years = 2019,
+      expenditure_concept = "total"
+    ),
+    class = "uscogdata_concept_not_aggregatable"
+  )
+})
+
+test_that("cog_peer_compare refuses expenditure_concept = 'total'", {
+  expect_error(
+    cog_peer_compare(
+      target_govid = "010000226085", peers = "010000226085",
+      category = "Police", years = 2019,
+      expenditure_concept = "total"
+    ),
+    class = "uscogdata_concept_not_aggregatable"
+  )
+})
+
+test_that("the refusal message names the fix and the reason", {
+  err <- tryCatch(
+    cog_geographic_rollup(govids = list(state = "010000226085"),
+                          category = "Police", years = 2019,
+                          expenditure_concept = "total"),
+    condition = function(e) e
+  )
+  msg <- paste(conditionMessage(err), collapse = " ")
+  expect_match(msg, "direct")
+  expect_match(msg, "double-count|double count")
+  expect_match(msg, "cog_geographic_rollup")
+
+  # Test that cog_peer_compare's message names its own function
+  err2 <- tryCatch(
+    cog_peer_compare(target_govid = "010000226085", peers = "010000226085",
+                     category = "Police", years = 2019,
+                     expenditure_concept = "total"),
+    condition = function(e) e
+  )
+  msg2 <- paste(conditionMessage(err2), collapse = " ")
+  expect_match(msg2, "direct")
+  expect_match(msg2, "double-count|double count")
+  expect_match(msg2, "cog_peer_compare")
+})
+
+test_that("both cross-government verbs still accept the direct default", {
+  expect_no_error(
+    cog_geographic_rollup(govids = list(state = "010000226085"),
+                          category = "Police", years = 2019)
+  )
+  expect_no_error(
+    cog_peer_compare(target_govid = "010000226085", peers = "010000226085",
+                     category = "Police", years = 2019)
+  )
+})
+
+test_that("provenance always records the expenditure concept", {
+  p <- cog_spending("010000226085", years = 2019, category = "Police")
+  d <- cog_spending("010000226085", years = 2019, category = "Police",
+                    expenditure_concept = "direct")
+  t <- cog_spending("010000226085", years = 2019, category = "Police",
+                    expenditure_concept = "total")
+  expect_equal(attr(p, "provenance")$expenditure_concept, "primary")
+  expect_equal(attr(d, "provenance")$expenditure_concept, "direct")
+  expect_equal(attr(t, "provenance")$expenditure_concept, "total")
+  # The note explains the non-obvious part: how legacy IG was assembled.
+  expect_true(nzchar(attr(t, "provenance")$expenditure_concept_note))
+  expect_true(is.na(attr(d, "provenance")$expenditure_concept_note) ||
+              !nzchar(attr(d, "provenance")$expenditure_concept_note))
+})
+
+test_that("the provenance schema documents expenditure_concept", {
+  sch <- jsonlite::fromJSON(
+    system.file("schemas", "provenance-v1.json", package = "uscogdata"),
+    simplifyVector = FALSE
+  )
+  expect_true("expenditure_concept" %in% names(sch$properties))
+})
+
+test_that("a firing suggestion names the intergovernmental counterpart recipe", {
+  # Corrections has no legacy leaf rows, so the coverage-gap suggestion fires;
+  # corrections_ig_local_combined is its IG counterpart.
+  r <- suppressMessages(
+    cog_spending("010000226085", years = c(2005, 2011), category = "Corrections")
+  )
+  sugg <- attr(r, "provenance")$suggestions
+  expect_gt(length(sugg), 0L)
+  ids <- vapply(sugg, function(s) s$recipe_id %||% "", character(1))
+  expect_true("corrections_combined" %in% ids)
+  ig <- unlist(lapply(sugg, function(s) s$ig_recipe_id))
+  expect_true("corrections_ig_local_combined" %in% ig)
+})
+
+test_that("no suggestion fires for a healthy query", {
+  r <- cog_spending("010000226085", years = 2019, category = "Police")
+  expect_length(attr(r, "provenance")$suggestions, 0L)
+})
+
+test_that("a mis-scoped cog_spending() call never attaches an M/L counterpart to a revenue-flavored recipe", {
+  # "IG Federal" is a revenue-only category (summary_categories maps it to
+  # B-prefixed component codes only; its recipes are ig_federal_b47_wide /
+  # ig_federal_b89_wide). A cog_spending() call scoped to it returns zero
+  # spending rows for every requested year -- there is no spending
+  # component in this category at all -- so the coverage-gap machinery
+  # fires for real (not hypothetically) even though this isn't the kind of
+  # format-boundary gap the recipe catalog is meant to signpost. This is
+  # exactly the live-corpus risk flagged in review: ig_federal_b47_wide's
+  # own component codes (B47/B94, suffixes {"47","94"}) are an EXACT
+  # suffix-set match for the expenditure recipe ige_local_m47_wide
+  # (M47/M94, same suffixes) -- a coincidence of reused digits, not a real
+  # Direct/Total pairing. The flow-family gate in
+  # .attach_ig_counterparts() must keep ig_recipe_id NULL here.
+  #
+  # Anchored on FL state government, not AL. Coverage is presence-based: a
+  # recipe is only suggested when its component codes have rows for the
+  # requested government-year. AL state's only FY2011 B47 cell was an
+  # explicit zero, which the corpus no longer stores after sparsification
+  # (SB194, cog_pipeline#64), so the recipe stopped being a candidate there.
+  # FL state carries a real FY2011 B47 amount, so this exercises the guard
+  # against a suggestion that genuinely fires.
+  r <- suppressMessages(
+    cog_spending("120000226351", years = c(2005, 2011), category = "IG Federal")
+  )
+  sugg <- attr(r, "provenance")$suggestions
+  expect_gt(length(sugg), 0L)
+  ids <- vapply(sugg, function(s) s$recipe_id %||% "", character(1))
+  expect_true("ig_federal_b47_wide" %in% ids)
+  ig <- unlist(lapply(sugg, function(s) s$ig_recipe_id))
+  expect_length(ig, 0L)
+})
+
+test_that("C1: 'total' on a legacy aggregate-only family reports the IG-only figure honestly, not as Direct + IG", {
+  # AL state government, Corrections, 2011. Measured pre-fix: 'total'
+  # returned $31,358,000 (the IG leg alone, on an aggregate-flagged M04/M05
+  # row) with 0 suggestions (the surviving IG row made the gap-detection
+  # machinery think the Direct leg was covered) and a note asserting
+  # "Total = Direct + intergovernmental" with no caveat. True Direct (via
+  # recipe = "corrections_combined") is $521,651,000 -- the IG-only figure
+  # is ~6% of it.
+  gov <- "010000226085"
+
+  d <- cog_spending(gov, years = 2011, category = "Corrections",
+                    expenditure_concept = "direct")
+  expect_equal(nrow(d), 0L)
+
+  t <- suppressMessages(cog_spending(
+    gov, years = 2011, category = "Corrections", expenditure_concept = "total"
+  ))
+  expect_equal(nrow(t), 1L)
+  expect_equal(t$spend_subtype, "intergovernmental")
+  expect_equal(t$amt_nominal, 31358000)
+
+  r <- cog_spending(gov, years = 2011, recipe = "corrections_combined")
+  expect_equal(r$amt_nominal, 521651000)
+
+  # C1(a): the recipe hints must fire for "total" exactly as they do for
+  # "direct" -- the surviving IG row must not be mistaken for Direct
+  # coverage.
+  prov <- attr(t, "provenance")
+  expect_gt(length(prov$suggestions), 0L)
+  ids <- vapply(prov$suggestions, function(s) s$recipe_id %||% "", character(1))
+  expect_true("corrections_combined" %in% ids)
+
+  # C1(b): the affected row's notes name a recovering recipe rather than
+  # staying silent, and the provenance carries a flag a downstream consumer
+  # (e.g. cog-api, which passes provenance through verbatim) can test.
+  expect_true(nzchar(t$notes))
+  expect_match(t$notes, "unavailable", fixed = TRUE)
+  expect_match(t$notes, "corrections_combined", fixed = TRUE)
+  expect_true(prov$expenditure_concept_direct_suppressed)
+
+  # The base "Total = Direct + IG" note must NOT stand unqualified when that
+  # arithmetic didn't actually happen for this row.
+  expect_match(prov$expenditure_concept_note, "NOTE", fixed = TRUE)
+  expect_match(prov$expenditure_concept_note,
+              "expenditure_concept_direct_suppressed", fixed = TRUE)
+})
+
+test_that("C1(b): expenditure_concept_direct_suppressed is FALSE when the Direct leg is present", {
+  d <- cog_spending("010000226085", years = 2019, category = "Police",
+                    expenditure_concept = "direct")
+  t <- cog_spending("010000226085", years = 2019, category = "Police",
+                    expenditure_concept = "total")
+  expect_false(isTRUE(attr(d, "provenance")$expenditure_concept_direct_suppressed))
+  expect_false(isTRUE(attr(t, "provenance")$expenditure_concept_direct_suppressed))
+  expect_false(any(nzchar(t$notes[t$spend_subtype == "intergovernmental"]) &
+                   grepl("unavailable", t$notes[t$spend_subtype == "intergovernmental"])))
+})
+
+# M/I fix: .detect_direct_suppressed() was equating "no Direct sibling row"
+# with "Direct was suppressed", but the dominant real cause is a government
+# that simply has no direct spending in that category -- correct, ordinary
+# data. The fix gates the flag (and its row note) on a harmonization recipe
+# ACTUALLY covering that exact (year, canonical_govid, category) triple.
+
+test_that("M/I: true positive, category supplied explicitly (unchanged behavior)", {
+  al <- "010000226085"
+  t_cat <- suppressMessages(cog_spending(
+    al, years = 2011, category = "Corrections", expenditure_concept = "total"
+  ))
+  expect_true(attr(t_cat, "provenance")$expenditure_concept_direct_suppressed)
+  expect_match(t_cat$notes, "corrections_combined", fixed = TRUE)
+  expect_match(t_cat$notes, "unavailable", fixed = TRUE)
+})
+
+test_that("M/I: true positive, category = NULL now also names the recipe (was the fallback bug)", {
+  # Root bug: .build_suggestions() short-circuits to list() when category is
+  # NULL, so the note previously always hit its "no covering recipe found"
+  # fallback here even though corrections_combined genuinely covers this row.
+  al <- "010000226085"
+  t_null <- suppressMessages(cog_spending(
+    al, years = 2011, category = NULL, expenditure_concept = "total"
+  ))
+  corr_row <- t_null[t_null$category %in% "Corrections", ]
+  expect_equal(nrow(corr_row), 1L)
+  expect_true(attr(t_null, "provenance")$expenditure_concept_direct_suppressed)
+  expect_match(corr_row$notes, "corrections_combined", fixed = TRUE)
+  expect_match(corr_row$notes, "unavailable", fixed = TRUE)
+  expect_false(grepl("no covering recipe found", corr_row$notes, fixed = TRUE))
+})
+
+test_that("M/I: false positive -- Virginia Education K-12 FY2019 total is NOT flagged", {
+  # States fund K-12 through school districts, so the Direct leg (E12/F12/
+  # G12) is genuinely, correctly zero -- not suppressed. Must not be flagged
+  # and must carry no suppression note.
+  va <- "510000227542"
+  t_va <- suppressMessages(cog_spending(
+    va, years = 2019, category = "Education K-12", expenditure_concept = "total"
+  ))
+  expect_equal(nrow(t_va), 1L)
+  expect_equal(t_va$spend_subtype, "intergovernmental")
+  expect_equal(t_va$amt_nominal, 8028179000)
+  expect_false(isTRUE(attr(t_va, "provenance")$expenditure_concept_direct_suppressed))
+  expect_false(nzchar(t_va$notes) && grepl("unavailable", t_va$notes))
+})
+
+test_that("M/I: false positive by construction -- 'Other Education' has no E/F/G code, never flagged", {
+  # "Other Education" maps only to M21/L21 in summary_categories -- there is
+  # no E/F/G code for it in this corpus at all, so no Direct-recovering
+  # recipe can exist and it must never be flagged, in any fixture year.
+  con <- uscogdata:::.ensure_session()
+  years_all <- DBI::dbGetQuery(con, "SELECT DISTINCT year FROM long ORDER BY year")$year
+  states <- DBI::dbGetQuery(con,
+    "SELECT DISTINCT canonical_govid FROM long WHERE type = 0")$canonical_govid
+  oe <- suppressMessages(cog_spending(
+    states, years = years_all, category = "Other Education",
+    expenditure_concept = "total"
+  ))
+  expect_false(isTRUE(attr(oe, "provenance")$expenditure_concept_direct_suppressed))
+  expect_false(any(nzchar(oe$notes) & grepl("unavailable", oe$notes)))
+})
+
+test_that("M/I: a clean FY2019 category = NULL total query flags far fewer than the pre-fix 32/50 states", {
+  con <- uscogdata:::.ensure_session()
+  states <- DBI::dbGetQuery(con,
+    "SELECT DISTINCT canonical_govid FROM long WHERE type = 0")$canonical_govid
+  r <- suppressMessages(cog_spending(
+    states, years = 2019, category = NULL, expenditure_concept = "total"
+  ))
+  ig <- r[r$spend_subtype == "intergovernmental", ]
+  flagged <- ig[nzchar(ig$notes) & grepl("unavailable", ig$notes), ]
+  expect_lt(length(unique(flagged$canonical_govid)), 32L)
+  # Every remaining flagged row must actually name a covering recipe --
+  # never the old no-recipe-found fallback.
+  expect_true(all(grepl("recipe = '", flagged$notes, fixed = TRUE)))
+  expect_false(any(grepl("no covering recipe found", flagged$notes, fixed = TRUE)))
+})
+
+test_that("C2: expenditure_concept = 'total' aborts on a corpus with no intergovernmental category rows", {
+  with_corpus_missing_ig_categories({
+    con <- uscogdata:::.ensure_session()
+    n <- DBI::dbGetQuery(con,
+      "SELECT COUNT(*) AS n FROM summary_categories WHERE LEFT(item_code, 1) IN ('M', 'L')"
+    )$n
+    expect_equal(n, 0)
+
+    err <- tryCatch(
+      cog_spending("010000226085", years = 2019, category = "Police",
+                  expenditure_concept = "total"),
+      condition = function(e) e
+    )
+    expect_s3_class(err, "uscogdata_ig_categories_unsupported")
+    msg <- conditionMessage(err)
+    expect_match(msg, "PR #59|predates", perl = TRUE)
+  })
+
+  # 'direct' is unaffected on the same corpus -- the guard is scoped to
+  # expenditure_concept = "total" only.
+  with_corpus_missing_ig_categories({
+    expect_no_error(
+      cog_spending("010000226085", years = 2019, category = "Police",
+                  expenditure_concept = "direct")
+    )
+  })
+})
+
+test_that("C2: expenditure_concept = 'total' still works on a corpus that DOES carry M/L category rows", {
+  expect_no_error(
+    cog_spending("010000226085", years = 2019, category = "Police",
+                expenditure_concept = "total")
+  )
+})
+
+test_that("I2: an intergovernmental (M/L) recipe never appears as its own top-level suggestion", {
+  # Task 1's M04/M05 category rows share the "Corrections" summary_categories
+  # category with the Direct-flavored E04/E05, so `corrections_ig_local_
+  # combined` (entirely M-prefixed) becomes a raw *candidate* in
+  # .build_suggestions()'s component_code-driven query. Following a
+  # "re-run with recipe = 'corrections_ig_local_combined'" hint on a plain
+  # cog_spending() call would silently return intergovernmental dollars
+  # under provenance$expenditure_concept = "direct". Task 6's gate
+  # (.attach_ig_counterparts()) already protects the *counterpart* lookup;
+  # this exercises that the candidate list itself is filtered too.
+  r <- suppressMessages(
+    cog_spending("010000226085", years = c(2005, 2011), category = "Corrections")
+  )
+  sugg <- attr(r, "provenance")$suggestions
+  ids <- vapply(sugg, function(s) s$recipe_id %||% "", character(1))
+  expect_true("corrections_combined" %in% ids)
+  expect_false("corrections_ig_local_combined" %in% ids)
+})
+
+test_that(".attach_ig_counterparts() never pairs a revenue-side recipe with its coincidental M/L suffix twin", {
+  # Broader version of the case above, run at the matching-helper level
+  # (the same level code review's pairwise enumeration was done at) rather
+  # than end-to-end: the fixture has no (govid, year) combination where
+  # cog_revenue() itself produces a covered gap for any B/C/D recipe, so an
+  # end-to-end repro for THIS specific set of recipes isn't reachable
+  # today. Each of these six recipes shares an exact suffix set with an
+  # M/L expenditure recipe purely by reused-digit coincidence:
+  #   ig_federal_b47_wide {"47","94"} == ige_local_m47_wide / ige_state_l47_wide
+  #   ig_federal_b89_wide {"89","91","92","93"} == ige_local_m89_wide / ige_state_l89_wide
+  #   ig_state_c47_wide   {"47","94"} == ige_local_m47_wide / ige_state_l47_wide
+  #   ig_state_c89_wide   {"89","91","92","93"} == ige_local_m89_wide / ige_state_l89_wide
+  #   ig_local_d47_wide   {"47","94"} == ige_local_m47_wide / ige_state_l47_wide
+  #   ig_local_d89_wide   {"89","91","92","93"} == ige_local_m89_wide / ige_state_l89_wide
+  # None of them may receive an ig_recipe_id under cog_revenue()'s own
+  # flow_prefixes, since M/L only ever pairs with the direct-expenditure
+  # (E/F/G) family.
+  con <- uscogdata:::.ensure_session()
+  fake_suggestion <- function(rid) {
+    list(recipe_id = rid, label = "x", available_years = c(1967L, 2023L),
+         hint = "h")
+  }
+  fake_suggestions <- lapply(
+    c("ig_federal_b47_wide", "ig_federal_b89_wide",
+      "ig_state_c47_wide", "ig_state_c89_wide",
+      "ig_local_d47_wide", "ig_local_d89_wide"),
+    fake_suggestion
+  )
+  out <- uscogdata:::.attach_ig_counterparts(
+    con, fake_suggestions, c("T", "A", "U", "B", "C", "D")
+  )
+  ig <- unlist(lapply(out, function(s) s$ig_recipe_id))
+  expect_length(ig, 0L)
+})

tests/testthat/test-expenditure-concepts.R

@@ -0,0 +1,112 @@
+# Madison walkthrough audit -- findings F-012, F-017, F-018.
+# Tracked as uscogdata#11. See docs/walkthroughs/FINDINGS.md in cog_explorer.
+#
+# The owner's settled three-concept model (2026-07-28):
+#   total   = primary + interest + intergovernmental transfers
+#   direct  = primary + interest          (Census's published Direct Expenditure)
+#   primary = direct minus debt service   (the NEW DEFAULT)
+# implemented by reclassifying on the crosswalk's `spend_type` column, NOT on
+# item-code first letters -- F-018 shows prefix `Y` carries both revenue
+# (Y01/Y02) and expenditure (Y05/Y06) codes, so no first-letter allowlist can
+# route them correctly.
+#
+# Fixture reproducibility: the finding's headline reconciliation is Madison
+# FY2022, where the corpus carries I89 = 46,609 (thousands) and Census's
+# published Direct Expenditure is $654,893,000 against cog_spending()'s
+# $608,284,000 (-7.1%). FY2022 is outside the bundled fixture's year window
+# (2011/2012/2019/2020), so the same invariant is asserted on FY2020, where the
+# fixture carries I89 = 27,704. Anyone running against the full corpus should
+# also check the FY2022 numbers above.
+
+test_that("expenditure concepts classify on spend_type, not item-code prefix", {
+  mad      <- "552025209777"   # MADISON CITY, WI
+  wi_state <- "550000227544"   # WISCONSIN (state government)
+
+  # -- F-012: `primary` is the new default, and equals today's E/F/G figure ---
+  primary <- cog_spending(govid = mad, years = 2020L)
+  expect_equal(attr(primary, "provenance")$expenditure_concept, "primary")
+  expect_equal(sum(primary$amt_nominal), 623347000)
+
+  # -- F-012: `direct` adds interest on long-term debt ------------------------
+  # Expected interest read from the RAW corpus, never through cog_spending(),
+  # which is the filter under test.
+  interest <- wt_raw_amt(mad, 2020L, prefixes = "I")
+  expect_equal(interest, 27704)                       # I89, in $1,000s
+
+  direct <- cog_spending(govid = mad, years = 2020L, expenditure_concept = "direct")
+  expect_equal(sum(direct$amt_nominal), 651051000)    # 623,347 + 27,704 thousands
+  expect_equal(sum(direct$amt_nominal) - sum(primary$amt_nominal), interest * 1000)
+  expect_true("I89" %in% wt_codes_included(direct))
+
+  # -- F-017: `total` carries Q12/Q18, state IG transfers to school districts --
+  # Wisconsin FY2019: Q12 = 6,431,530 and Q18 = 533,391 (thousands). Today
+  # neither verb's flow_prefixes contains "Q", so both are dropped from the one
+  # concept that is supposed to include intergovernmental transfers.
+  ig_expected <- wt_raw_amt(wi_state, 2019L, prefixes = c("M", "L", "Q"))
+  expect_equal(ig_expected, 11609814)                 # M 4,644,893 + Q 6,964,921
+
+  wi_direct <- cog_spending(govid = wi_state, years = 2019L,
+                            expenditure_concept = "direct")
+  wi_total  <- cog_spending(govid = wi_state, years = 2019L,
+                            expenditure_concept = "total")
+
+  # total - direct is exactly the intergovernmental component. Asserted as a
+  # delta rather than a grand total so this stays correct however the J and Y
+  # families land inside `primary`.
+  expect_equal(sum(wi_total$amt_nominal) - sum(wi_direct$amt_nominal),
+               ig_expected * 1000)
+  expect_true(all(c("Q12", "Q18") %in% wt_codes_included(wi_total)))
+
+  # -- F-018: prefix Y splits revenue from expenditure, by spend_type ---------
+  # Y01/Y02 are Insurance Trust revenue; Y05/Y06 are Insurance Trust benefit
+  # payments. All four share the first letter `Y`, so no first-letter allowlist
+  # can route them. The proof that classification is crosswalk-keyed:
+  # Y05 lands in `total` spending (insurance_benefits is inside `direct`),
+  # while Y01 -- same prefix -- is classified `revenue` by the crosswalk and
+  # therefore can never appear in a spending result.
+  #
+  # Per the owner's 2026-07-30 ruling (#11 DoD item 4 vs #12), cog_revenue()'s
+  # DEFAULT stays Census General Revenue and so excludes insurance-trust
+  # revenue; Y01's revenue-side classification is asserted against the
+  # crosswalk itself, not the default call. Surfacing Y01 through an explicit
+  # revenue concept argument is uscogdata#12.
+  wi_revenue <- cog_revenue(govid = wi_state, years = 2019L)
+  spend_codes <- wt_codes_included(wi_total)
+  rev_codes   <- wt_codes_included(wi_revenue)
+
+  expect_true("Y05" %in% spend_codes)
+  expect_false("Y05" %in% rev_codes)
+  expect_false("Y01" %in% spend_codes)
+  expect_false("Y01" %in% rev_codes)   # default = general revenue (#12 ruling)
+
+  con <- uscogdata:::.ensure_session()
+  y_class <- DBI::dbGetQuery(con,
+    "SELECT item_code, category_type, spend_subtype, revenue_subtype
+     FROM summary_categories WHERE item_code IN ('Y01', 'Y05')")
+  expect_equal(y_class$category_type[y_class$item_code == "Y01"], "revenue")
+  expect_equal(y_class$revenue_subtype[y_class$item_code == "Y01"], "insurance_trust")
+  expect_equal(y_class$category_type[y_class$item_code == "Y05"], "expenditure")
+  expect_equal(y_class$spend_subtype[y_class$item_code == "Y05"], "insurance_benefits")
+})
+
+test_that("no balance code or category ever reaches a spending or revenue result (uscogdata#25)", {
+  # Stocks are not flows. The crosswalk's balance codes (W/X/Y/Z fund
+  # balances) share first letters with flow codes, so this could never be
+  # guaranteed under prefix classification; under crosswalk membership it
+  # falls out structurally -- asserted here at the verb level, on a
+  # government-year the fixture gives real balance rows (Wisconsin carries
+  # Y07/Y08/Y21/Y61-type balances in FY2019).
+  wi_state <- "550000227544"
+  con <- uscogdata:::.ensure_session()
+  balance <- DBI::dbGetQuery(con,
+    "SELECT item_code, category FROM summary_categories WHERE category_type = 'balance'")
+  expect_gt(nrow(balance), 0L)
+
+  spend <- cog_spending(wi_state, 2019L, expenditure_concept = "total")
+  rev   <- cog_revenue(wi_state, 2019L)
+
+  expect_false(any(spend$category %in% balance$category))
+  expect_false(any(rev$category %in% balance$category))
+  expect_length(intersect(wt_codes_included(spend), balance$item_code), 0L)
+  expect_length(intersect(wt_codes_included(rev), balance$item_code), 0L)
+})

tests/testthat/test-explain.R

@@ -1,6 +1,6 @@
 test_that("cog_explain prints verb header and target", {
   skip_if_no_corpus()
-  r <- cog_spending("101006006", 2020L, "Corrections")
+  r <- cog_spending("121011212191", 2020L, "Corrections")
   # cli writes to stderr; capture both stdout and message streams.
   txt <- paste(c(
     capture.output(cog_explain(r)),
@@ -8,19 +8,19 @@ test_that("cog_explain prints verb header and target", {
   ), collapse = "\n")
   expect_true(grepl("cog_spending", txt))
   expect_true(grepl("Corrections", txt))
-  expect_true(grepl("101006006", txt))
+  expect_true(grepl("121011212191", txt))
 })
 
 test_that("cog_explain format='list' returns structured provenance", {
   skip_if_no_corpus()
-  r <- cog_spending("101006006", 2020L, "Corrections")
+  r <- cog_spending("121011212191", 2020L, "Corrections")
   prov <- cog_explain(r, format = "list")
   expect_identical(prov, attr(r, "provenance"))
 })
 
 test_that("cog_explain returns result invisibly for chaining", {
   skip_if_no_corpus()
-  r <- cog_spending("101006006", 2020L, "Corrections")
+  r <- cog_spending("121011212191", 2020L, "Corrections")
   res <- withVisible(cog_explain(r))
   expect_false(res$visible)
   expect_identical(res$value, r)
@@ -30,3 +30,91 @@ test_that("cog_explain errors on non-verb input", {
   df <- tibble::tibble(a = 1)
   expect_error(cog_explain(df), "provenance")
 })
+
+test_that("cog_explain prints basis + harmonization block", {
+  skip_if_no_corpus()
+  r <- cog_spending("121011212191", 2020L, "Corrections")
+  txt <- paste(c(
+    capture.output(cog_explain(r)),
+    capture.output(cog_explain(r), type = "message")
+  ), collapse = "\n")
+  expect_true(grepl("Basis: harmonized", txt))
+  expect_true(grepl("Harmonization", txt))
+  expect_true(grepl("Excluded 0 row", txt))
+})
+
+test_that("cog_explain prints a Recipe section for recipe = results", {
+  skip_if_no_corpus()
+  r <- cog_spending("121011212191", c(2011L, 2012L), recipe = "corrections_combined")
+  txt <- paste(c(
+    capture.output(cog_explain(r)),
+    capture.output(cog_explain(r), type = "message")
+  ), collapse = "\n")
+  expect_true(grepl("Recipe", txt))
+  expect_true(grepl("corrections_combined", txt))
+  expect_true(grepl("E04", txt))
+  expect_true(grepl("E05", txt))
+})
+
+test_that("cog_explain prints a Suggestions section when the provenance has one", {
+  skip_if_no_corpus()
+  r <- suppressMessages(
+    cog_spending("121011212191", c(2011L, 2012L), category = "Corrections")
+  )
+  txt <- paste(c(
+    capture.output(cog_explain(r)),
+    capture.output(cog_explain(r), type = "message")
+  ), collapse = "\n")
+  expect_true(grepl("Suggestions", txt))
+  expect_true(grepl("corrections_combined", txt))
+  expect_true(grepl("re-run with recipe", txt))
+})
+
+test_that("cog_explain prints the expenditure concept (I1)", {
+  skip_if_no_corpus()
+  d <- cog_spending("010000226085", years = 2019, category = "Police")
+  t <- cog_spending("010000226085", years = 2019, category = "Police",
+                    expenditure_concept = "total")
+  txt_d <- paste(c(
+    capture.output(cog_explain(d)),
+    capture.output(cog_explain(d), type = "message")
+  ), collapse = "\n")
+  txt_t <- paste(c(
+    capture.output(cog_explain(t)),
+    capture.output(cog_explain(t), type = "message")
+  ), collapse = "\n")
+  expect_true(grepl("Concept: primary", txt_d))
+  expect_true(grepl("Concept: total", txt_t))
+})
+
+test_that("cog_explain surfaces the C1(b) direct-suppressed flag as a warning", {
+  skip_if_no_corpus()
+  t <- suppressMessages(cog_spending(
+    "010000226085", years = 2011, category = "Corrections",
+    expenditure_concept = "total"
+  ))
+  expect_true(attr(t, "provenance")$expenditure_concept_direct_suppressed)
+  txt <- paste(c(
+    capture.output(cog_explain(t)),
+    capture.output(cog_explain(t), type = "message")
+  ), collapse = "\n")
+  expect_true(grepl("Direct leg unavailable", txt))
+})
+
+test_that("cog_explain prints denominator + popyear_range + counts", {
+  skip_if_no_corpus()
+  with_fixture_corpus({
+    r <- cog_spending("121011212191", years = 2019:2020,
+                      category = "Police", per_capita = TRUE)
+    out <- paste(c(
+      capture.output(cog_explain(r)),
+      capture.output(cog_explain(r), type = "message")
+    ), collapse = "\n")
+    expect_true(grepl("Census F-33", out))
+    expect_true(grepl("popyear", out, ignore.case = TRUE))
+    expect_true(grepl("census_f33", out))
+    # popyear_range should render as 4-digit calendar years, not raw 2-digit
+    expect_true(grepl("2019-2020", out))
+    expect_false(grepl("popyear range: 19-20", out, fixed = TRUE))
+  })
+})

tests/testthat/test-fixture-vintage.R

@@ -0,0 +1,112 @@
+# tests/testthat/test-fixture-vintage.R
+#
+# The bundled fixture is a slice of a real cog_pipeline publish tree, and
+# every test in this package -- plus the whole cog-api suite -- runs against
+# it. When the published corpus changes shape and the fixture does not, both
+# suites stay green against a corpus that no longer exists (uscogdata#18).
+#
+# These tests pin the structural facts that distinguish the current published
+# vintage from its predecessor, so a stale fixture fails loudly instead of
+# passing quietly. They assert shape, never dollar values: re-running
+# data-raw/regenerate_fixture_corpus.R against a newer publish tree should
+# keep them green.
+
+# Open a bare DuckDB connection on the fixture's parquet files. Deliberately
+# not the package session: these assertions are about what the fixture
+# CONTAINS, and routing them through the reader's own views would let a
+# filter hide the very absence being checked.
+fixture_query <- function(sql, ...) {
+  con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  path <- function(rel) {
+    sprintf("read_parquet(%s)",
+            DBI::dbQuoteString(con, file.path(fixture_corpus_path(), rel)))
+  }
+  DBI::dbGetQuery(con, do.call(sprintf, c(list(sql), lapply(c(...), path))))
+}
+
+test_that("fixture ships every metadata table the publish tree does", {
+  skip_if_no_corpus()
+  # representation/code_set are what make a sparse corpus interpretable; a
+  # fixture without them predates sparsification (cog_pipeline#64).
+  expected <- c(
+    "canonical_alias.parquet", "canonical_fips_xwalk.parquet",
+    "census_collection_coverage.parquet", "code_set.parquet",
+    "harmonization_map.parquet", "harmonization_recipes.parquet",
+    "lineage_events.parquet", "representation.parquet",
+    "series_breaks.parquet", "summary_categories.parquet"
+  )
+  on_disk <- basename(list.files(
+    file.path(fixture_corpus_path(), "data"), pattern = "\\.parquet$"
+  ))
+  expect_true(all(expected %in% on_disk))
+
+  # The manifest must list them too -- consumers read the manifest, not ls().
+  in_manifest <- with_fixture_corpus(
+    basename(vapply(cog_manifest()$files$metadata, function(f) f$path, character(1)))
+  )
+  expect_true(all(expected %in% in_manifest))
+})
+
+test_that("fixture carries the dense/sparse representation contract", {
+  skip_if_no_corpus()
+  rep <- fixture_query(
+    "SELECT year, representation, absence_means FROM %s
+     WHERE year IN (2011, 2012, 2019, 2020) ORDER BY year",
+    "data/representation.parquet"
+  )
+  expect_equal(nrow(rep), 4L)
+  expect_equal(rep$representation, c("dense_source", rep("sparse_source", 3L)))
+  expect_equal(rep$absence_means, c("census_zero", rep("not_reported", 3L)))
+})
+
+test_that("the fixture's wide era is sparse, not zero-padded", {
+  skip_if_no_corpus()
+  # FY2011 is a dense_source year: the corpus publishes only the cells Census
+  # reported non-zero, and an absent cell means Census published $0. Before
+  # sparsification this partition was 2,864,212 rows, ~83% of them explicit
+  # zeros. A single explicit zero here means the fixture predates the change.
+  zeros_2011 <- fixture_query(
+    "SELECT COUNT(*) AS n FROM %s WHERE amt = 0",
+    "data/long/year=2011/part-0.parquet"
+  )$n
+  expect_equal(zeros_2011, 0L)
+
+  # The modern era is a different regime: a reported zero there is real data
+  # (the government filed $0), so zeros legitimately survive and must not be
+  # asserted away.
+  expect_gt(
+    fixture_query("SELECT COUNT(*) AS n FROM %s", "data/long/year=2012/part-0.parquet")$n,
+    0L
+  )
+})
+
+test_that("code_set covers every fixture year with the reader-spec columns", {
+  skip_if_no_corpus()
+  cs <- fixture_query(
+    "SELECT * FROM %s WHERE year IN (2011, 2012, 2019, 2020)",
+    "data/code_set.parquet"
+  )
+  expect_true(all(
+    c("code_set_id", "year", "type", "item_code", "is_aggregate", "n_units")
+    %in% names(cs)
+  ))
+  expect_setequal(unique(cs$year), c(2011L, 2012L, 2019L, 2020L))
+})
+
+test_that("every flow code carrying dollars has a category, J-prefix included", {
+  skip_if_no_corpus()
+  # The J (assistance/benefit) codes were uncategorised until the crosswalk
+  # completion shipped (cog_pipeline#60/#65, J19 held back until #64's
+  # duplication fix landed). Their absence is how a pre-crosswalk fixture
+  # gives itself away.
+  j <- fixture_query(
+    "SELECT item_code, category, category_type, spend_subtype FROM %s
+     WHERE LEFT(item_code, 1) = 'J' ORDER BY item_code",
+    "data/summary_categories.parquet"
+  )
+  expect_true("J19" %in% j$item_code)
+  expect_true(all(j$category_type == "expenditure"))
+  expect_true(all(j$spend_subtype == "assistance"))
+  expect_false(any(is.na(j$category)))
+})

tests/testthat/test-gov-search-literal-match.R

@@ -0,0 +1,57 @@
+# Madison walkthrough audit -- finding F-025. Tracked as uscogdata#16.
+# See docs/walkthroughs/FINDINGS.md in cog_explorer.
+#
+# cog_gov_search()'s UTILITY mode interpolates `name` into
+#   regexp_matches(gov_name, <name>, 'i')
+# unescaped (R/search.R:102), while BASKET mode in the same file already routes
+# it through .escape_regex() (R/search.R:307) with the comment "so `name` is
+# treated as a literal substring". Two failure modes result:
+#   correctness -- a real government cannot be found by its own exact name, and
+#                  a single "." matches everything (HTTP 200 both ways via the API);
+#   robustness  -- malformed regex reaches the engine and errors, which cog-api
+#                  surfaces as a 500, reachable by typing a real name one
+#                  character at a time.
+#
+# NOT asserted here: the finding's `q=St. Louis` example. Under correct literal
+# matching that search still returns 0 rows, because the stored name is
+# "ST LOUIS CITY" with no period -- it demonstrates today's over-matching
+# semantics, not a row the fix makes findable.
+
+test_that("cog_gov_search() matches name literally, not as an unescaped regex", {
+
+  # -- correctness (1): a government must be findable by its own exact name ---
+  # FREDONIA (BRISCOE) CITY is real; today the parentheses are read as regex
+  # grouping, so its own complete name matches nothing.
+  fredonia <- cog_gov_search(name = "FREDONIA (BRISCOE) CITY")
+  expect_equal(nrow(fredonia), 1L)
+  expect_equal(fredonia$canonical_govid, "052117184386")
+  expect_equal(cog_gov_search(name = "FREDONIA (BRISCOE)")$canonical_govid,
+               "052117184386")
+
+  # -- correctness (2): a metacharacter must not become a wildcard ------------
+  # No Wisconsin city or village name contains a literal period -- established
+  # against the raw registry below, NOT through the verb under test. A literal
+  # search for "." must therefore return nothing; today it returns all 608.
+  con <- DBI::dbConnect(duckdb::duckdb())
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  xwalk <- paste0(sub("/$", "", Sys.getenv("USCOGDATA_URL")),
+                  "/data/canonical_fips_xwalk.parquet")
+  with_dot <- DBI::dbGetQuery(con, paste0(
+    "SELECT COUNT(*) n FROM read_parquet('", xwalk, "') ",
+    "WHERE fips_state = '55' AND govs_type = 2 AND gov_name LIKE '%.%'"))
+  expect_equal(as.integer(with_dot$n[[1]]), 0L)
+
+  expect_equal(nrow(cog_gov_search(name = ".", state = "WI", type = "city")), 0L)
+  expect_equal(nrow(cog_gov_search(name = "M.dison", state = "WI", type = "city")), 0L)
+  expect_equal(nrow(cog_gov_search(name = "Mad(i|o)son", state = "WI", type = "city")), 0L)
+
+  # A metacharacter-free name still resolves exactly as before.
+  expect_equal(nrow(cog_gov_search(name = "Madison", state = "WI", type = "city")), 1L)
+
+  # -- robustness: malformed pattern text returns no rows, and does not error --
+  # "[" alone, and "Athens-Clarke County (bal" -- an in-progress substring of
+  # ATHENS-CLARKE COUNTY (BALANCE), a real government -- both currently raise
+  # (DuckDB: "Invalid Input Error: missing ]").
+  expect_equal(nrow(cog_gov_search(name = "[")), 0L)
+  expect_equal(nrow(cog_gov_search(name = "Athens-Clarke County (bal")), 0L)
+})

tests/testthat/test-manifest.R

@@ -0,0 +1,165 @@
+# tests/testthat/test-manifest.R
+#
+# Tests for the guards on .fetch_or_cache_manifest() and cog_open() that
+# protect users from silent failures when USCOGDATA_URL is misconfigured
+# or returns non-JSON content.
+
+test_that("cog_open aborts with actionable error when URL is the placeholder default", {
+  uscogdata:::cog_close()
+  on.exit(uscogdata:::cog_close(), add = TRUE)
+
+  placeholder <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/"
+  withr::with_envvar(c(USCOGDATA_URL = placeholder), {
+    expect_error(
+      uscogdata:::cog_open(),
+      class = "uscogdata_url_not_configured"
+    )
+  })
+})
+
+test_that("placeholder guard fires for any URL containing the sentinel token", {
+  uscogdata:::cog_close()
+  on.exit(uscogdata:::cog_close(), add = TRUE)
+
+  # Sentinel detection should be substring-based — covers any host that still
+  # has REPLACE_WITH_SHARE_TOKEN baked in (default or partial user edit).
+  withr::with_envvar(c(USCOGDATA_URL = "https://other.example/s/REPLACE_WITH_SHARE_TOKEN/x/"), {
+    expect_error(
+      uscogdata:::cog_open(),
+      class = "uscogdata_url_not_configured"
+    )
+  })
+})
+
+test_that("placeholder guard error names both env var and option as remediation", {
+  uscogdata:::cog_close()
+  on.exit(uscogdata:::cog_close(), add = TRUE)
+
+  placeholder <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/"
+  withr::with_envvar(c(USCOGDATA_URL = placeholder), {
+    msg <- tryCatch(uscogdata:::cog_open(), error = conditionMessage)
+    expect_match(msg, "USCOGDATA_URL", fixed = TRUE)
+    expect_match(msg, "uscogdata.url", fixed = TRUE)
+  })
+})
+
+test_that("local manifest containing HTML produces uscogdata_invalid_manifest, not raw parse error", {
+  uscogdata:::cog_close()
+  on.exit(uscogdata:::cog_close(), add = TRUE)
+
+  tmp <- withr::local_tempdir()
+  writeLines(
+    c("<html>", "  <head><title>Welcome to our server</title></head>", "</html>"),
+    file.path(tmp, "manifest.json")
+  )
+
+  withr::with_envvar(c(USCOGDATA_URL = paste0(tmp, "/")), {
+    err <- expect_error(
+      uscogdata:::cog_open(),
+      class = "uscogdata_invalid_manifest"
+    )
+    expect_match(conditionMessage(err), "manifest", ignore.case = TRUE)
+  })
+})
+
+test_that("remote manifest fetch does not poison cache when response is HTML", {
+  uscogdata:::cog_close()
+  on.exit(uscogdata:::cog_close(), add = TRUE)
+
+  tmp_cache <- withr::local_tempdir()
+  cache_path <- file.path(tmp_cache, "manifest.json")
+
+  # Pretend the cache already exists with stale-but-fresh-by-mtime HTML
+  # (simulating a previous poisoned write from the old behavior). When the
+  # fetcher sees invalid JSON in the cache, it must refetch rather than
+  # silently returning a parse error to the caller.
+  writeLines("<html>poisoned</html>", cache_path)
+  Sys.setFileTime(cache_path, Sys.time())  # ensure within TTL
+
+  # We don't have a live HTTP fixture here, so the refetch will fail at the
+  # network layer — but the failure should NOT be a jsonlite parse error on
+  # the cached HTML; it should be a network-level httr2 error. The cache
+  # file itself must remain untouched (no atomic-write half-states).
+  withr::with_envvar(
+    c(
+      USCOGDATA_URL = "https://invalid.localhost.uscogdata.test/",
+      USCOGDATA_CACHE_DIR = tmp_cache
+    ),
+    {
+      err <- tryCatch(uscogdata:::cog_open(), error = identity)
+      expect_s3_class(err, "error")
+      # Must not be a JSON lexical error on HTML.
+      expect_false(grepl("lexical error", conditionMessage(err), fixed = TRUE))
+    }
+  )
+
+  # Atomic write contract: no stray tmp files left behind in cache_dir.
+  expect_length(
+    list.files(tmp_cache, pattern = "manifest\\.json\\.tmp"),
+    0L
+  )
+})
+
+test_that("cog_manifest returns the active session's parsed manifest", {
+  with_fixture_corpus({
+    m <- cog_manifest()
+    expect_type(m, "list")
+    expect_true(m$schema_version >= 4L)
+    yrs <- vapply(m$files$long_partitions, function(p) as.integer(p$year),
+                  integer(1))
+    expect_setequal(yrs, c(2011L, 2012L, 2019L, 2020L))
+  })
+})
+
+test_that(".validate_schema accepts schema_version 4 through 7, rejects others", {
+  expect_silent(uscogdata:::.validate_schema(list(schema_version = 4L)))
+  expect_silent(uscogdata:::.validate_schema(list(schema_version = 5L)))
+  # v6 = FIPS geography harmonization (2026-07-22): _code -> _asof rename +
+  # cog_legacy_* columns (26 -> 28 cols). This package references none of the
+  # renamed columns and its geography comes from the xwalk, so v6 is accepted
+  # without behavioural change -- see .validate_schema()'s note.
+  expect_silent(uscogdata:::.validate_schema(list(schema_version = 6L)))
+  # v7 = `data_year` APPENDED as column 29 (cog_pipeline #80, 2026-08-03), the
+  # most recent fiscal year contributing to a collapsed key. Appended, never
+  # inserted: canonical_govid stays at position 26, so nothing this package
+  # reads shifts. Verified against the real v7 corpus before widening the
+  # allow-list -- cog_spending()/cog_balances() return correctly for FY2024 AND
+  # for FY2012, so the new column is inert here.
+  expect_silent(uscogdata:::.validate_schema(list(schema_version = 7L)))
+  expect_error(
+    uscogdata:::.validate_schema(list(schema_version = 3L)),
+    "schema_version"
+  )
+  # The upper bound still has to be ENFORCED, not just moved. Without this the
+  # test would no longer prove that an unknown future schema is refused, and a
+  # v8 corpus with a genuinely breaking change would sail through.
+  expect_error(
+    uscogdata:::.validate_schema(list(schema_version = 8L)),
+    "schema_version"
+  )
+})
+
+test_that("cog_open succeeds against a doctored schema_version 4 corpus (dual-accept)", {
+  skip_if_no_corpus()
+  with_doctored_schema_version(4L, {
+    con <- cog_open()
+    expect_true(DBI::dbIsValid(con))
+    expect_equal(as.integer(cog_manifest()$schema_version), 4L)
+
+    # Core (pre-Phase-R2) views must still register on a v4 corpus.
+    views <- DBI::dbGetQuery(con,
+      "SELECT table_name FROM information_schema.tables
+       WHERE table_schema = 'main' AND table_type = 'VIEW'"
+    )$table_name
+    expect_true(all(c("spending_annotated", "revenue_annotated") %in% views))
+
+    # Schema-v5-only harmonization views must NOT register on a v4 corpus:
+    # their parquet sources don't exist there and DuckDB's read_parquet()
+    # errors eagerly at CREATE VIEW time for a missing file/glob, so
+    # .register_views() gates these on manifest$schema_version >= 5.
+    expect_false(any(c(
+      "spending_long_harmonized", "spending_annotated_harmonized",
+      "harmonization_recipes", "harmonization_map", "series_breaks_pq"
+    ) %in% views))
+  })
+})

tests/testthat/test-mirror.R

@@ -61,7 +61,7 @@ test_that("cog_mirror reads back via a fresh session against the mirror", {
   cog_close()
   options(uscogdata.url = paste0(normalizePath(tmp), "/"))
 
-  r <- cog_spending("101006006", 2020L, "Corrections")
+  r <- cog_spending("121011212191", 2020L, "Corrections")
   expect_gt(nrow(r), 0L)
-  expect_equal(unique(r$canonical_govid), "101006006")
+  expect_equal(unique(r$canonical_govid), "121011212191")
 })