feat: limit/offset on cog_gov_search() and cog_balances() (#57)
verbs were left materializing everything and slicing in R -- the pattern behind the 2026-08-06 production incident. cog_gov_search() had no LIMIT at all, so an unfiltered call returns the entire 40,336-row crosswalk. Extracted the #39 machinery into R/pagination.R first (.validate_pagination(), .paginate_sql(), .take_pagination_total()) rather than growing a third inline copy: three definitions of what total_rows means is three places for it to drift. Conflict refusals stay at the call sites because each verb's conflict set differs. .verb_spendrev() now uses the shared helpers and is unchanged in behaviour. The empty-page fallback query is now passed as a thunk, so the unpaginated SQL is only BUILT when an offset actually lands past the end instead of on every paged call. Two things #57 did not anticipate: - cog_gov_search()'s ORDER BY was not a total order. population_acs DESC NULLS LAST leaves ties -- and the whole NULL block -- in scan order, so two requests can order them differently and a paged sweep duplicates one row while dropping another. Added canonical_govid as tiebreaker. Unpaginated output changes only in the relative order of already-tied rows. - Basket mode returns one resolved row per requested name plus a sidecar covering all of them, so a page of it is not a page of anything the caller asked for. Refused with uscogdata_basket_pagination_conflict rather than silently ignoring the arguments. Both default to NULL, so cog-api adopts them behind its existing formals() probe with no lockstep deploy. Suite: 1067 passed, 0 failed, 0 warnings (2 pre-existing live-corpus skips).
This commit is contained in:
+41
-2
@@ -47,6 +47,12 @@
|
||||
#' @param recipe Optional harmonization recipe id (see [cog_recipes()]).
|
||||
#' `"cash_securities_z77_wide"` and `"cash_securities_z78_wide"` bridge the
|
||||
#' wide era to the modern one.
|
||||
#' @param limit Maximum number of result rows to return, pushed into the SQL
|
||||
#' rather than applied after materializing every row. `NULL` (default)
|
||||
#' returns everything. Cannot be combined with `recipe` -- see `offset` and
|
||||
#' `total_rows`.
|
||||
#' @param offset Rows to skip before `limit` starts counting (0-based).
|
||||
#' Ignored if `limit` is `NULL`; defaults to `0L` when `limit` is set.
|
||||
#'
|
||||
#' @return Tibble with columns `year`, `canonical_govid`, `gov_name`,
|
||||
#' `balance_subtype`, `category`, `amt_nominal`, `codes_included`,
|
||||
@@ -64,11 +70,16 @@
|
||||
#' and `truncated` (the observed subtypes whose coverage falls short of the
|
||||
#' requested years). `expenditure_concept`/`revenue_concept` are `NA` --
|
||||
#' holdings are a stock, not a flow, so neither concept vocabulary applies.
|
||||
#'
|
||||
#' When `limit` is set, also carries a `total_rows` attribute: the full
|
||||
#' unpaginated row count, computed by the same query (`COUNT(*) OVER()`)
|
||||
#' rather than a second scan.
|
||||
#' @export
|
||||
cog_balances <- function(govid = NULL, years, category = NULL,
|
||||
per_capita = FALSE, adjust_to_year = NULL,
|
||||
basis = c("harmonized", "raw"), recipe = NULL,
|
||||
state = NULL, type = NULL) {
|
||||
state = NULL, type = NULL,
|
||||
limit = NULL, offset = NULL) {
|
||||
call <- match.call()
|
||||
basis <- match.arg(basis, c("harmonized", "raw"))
|
||||
# Coerce FIRST, validate second: .validate_verb_inputs() asserts
|
||||
@@ -91,6 +102,22 @@ cog_balances <- function(govid = NULL, years, category = NULL,
|
||||
# validator's own doc comment for the incident that made that matter.
|
||||
.validate_verb_inputs(govid, years, category, per_capita, adjust_to_year,
|
||||
recipe)
|
||||
|
||||
# Same semantics as the money verbs (R/pagination.R). Only the `recipe`
|
||||
# conflict applies here: cog_balances() has no `complete` argument, and a
|
||||
# recipe's result comes from .run_recipe()'s own query, which pagination is
|
||||
# not wired into.
|
||||
paging <- .validate_pagination(limit, offset)
|
||||
limit <- paging$limit
|
||||
offset <- paging$offset
|
||||
if (!is.null(limit) && !is.null(recipe)) {
|
||||
cli::cli_abort(c(
|
||||
"`limit`/`offset` cannot be combined with `recipe`.",
|
||||
"i" = "A recipe's result comes from a separate query (`.run_recipe()`) that pagination is not wired into yet.",
|
||||
"*" = "Drop `limit`/`offset`, or drop `recipe`."
|
||||
), class = "uscogdata_recipe_pagination_conflict")
|
||||
}
|
||||
|
||||
years <- as.integer(years)
|
||||
if (!is.null(adjust_to_year)) adjust_to_year <- as.integer(adjust_to_year)
|
||||
|
||||
@@ -108,6 +135,7 @@ cog_balances <- function(govid = NULL, years, category = NULL,
|
||||
manifest <- .uscogdata_env$manifest
|
||||
recipe_block <- NULL
|
||||
category_for_prov <- category
|
||||
total_rows <- NULL # set below only when limit is non-NULL (non-recipe path)
|
||||
|
||||
if (!is.null(recipe)) {
|
||||
.require_schema_v5(con, manifest, "recipe =")
|
||||
@@ -125,8 +153,18 @@ cog_balances <- function(govid = NULL, years, category = NULL,
|
||||
} else {
|
||||
sql <- .build_verb_sql("balance_annotated", "balance_subtype",
|
||||
cohort, years, category,
|
||||
ig_view = NULL, subtype_scope = NULL)
|
||||
ig_view = NULL, subtype_scope = NULL,
|
||||
limit = limit, offset = offset)
|
||||
result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
|
||||
if (!is.null(limit)) {
|
||||
paged <- .take_pagination_total(result, con, function() {
|
||||
.build_verb_sql("balance_annotated", "balance_subtype",
|
||||
cohort, years, category,
|
||||
ig_view = NULL, subtype_scope = NULL)
|
||||
})
|
||||
result <- paged$result
|
||||
total_rows <- paged$total_rows
|
||||
}
|
||||
}
|
||||
|
||||
# Order matters (matches .verb_spendrev()): per-capita first, so
|
||||
@@ -158,6 +196,7 @@ cog_balances <- function(govid = NULL, years, category = NULL,
|
||||
.emit_balance_caveats(prov$balance_caveats)
|
||||
|
||||
attr(result, "provenance") <- prov
|
||||
if (!is.null(limit)) attr(result, "total_rows") <- total_rows
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user