feat: limit/offset on cog_gov_search() and cog_balances() (#57)
verbs were left materializing everything and slicing in R -- the pattern behind the 2026-08-06 production incident. cog_gov_search() had no LIMIT at all, so an unfiltered call returns the entire 40,336-row crosswalk. Extracted the #39 machinery into R/pagination.R first (.validate_pagination(), .paginate_sql(), .take_pagination_total()) rather than growing a third inline copy: three definitions of what total_rows means is three places for it to drift. Conflict refusals stay at the call sites because each verb's conflict set differs. .verb_spendrev() now uses the shared helpers and is unchanged in behaviour. The empty-page fallback query is now passed as a thunk, so the unpaginated SQL is only BUILT when an offset actually lands past the end instead of on every paged call. Two things #57 did not anticipate: - cog_gov_search()'s ORDER BY was not a total order. population_acs DESC NULLS LAST leaves ties -- and the whole NULL block -- in scan order, so two requests can order them differently and a paged sweep duplicates one row while dropping another. Added canonical_govid as tiebreaker. Unpaginated output changes only in the relative order of already-tied rows. - Basket mode returns one resolved row per requested name plus a sidecar covering all of them, so a page of it is not a page of anything the caller asked for. Refused with uscogdata_basket_pagination_conflict rather than silently ignoring the arguments. Both default to NULL, so cog-api adopts them behind its existing formals() probe with no lockstep deploy. Suite: 1067 passed, 0 failed, 0 warnings (2 pre-existing live-corpus skips).
This commit is contained in:
+14
-44
@@ -398,17 +398,10 @@ cog_spending <- function(govid = NULL, years, category = NULL,
|
||||
# up front rather than silently ignored: complete = TRUE fills a grid over
|
||||
# the FULL requested (year, category) space, and a recipe's result comes
|
||||
# from .run_recipe()'s own query, which this function does not touch.
|
||||
paging <- .validate_pagination(limit, offset)
|
||||
limit <- paging$limit
|
||||
offset <- paging$offset
|
||||
if (!is.null(limit)) {
|
||||
limit <- as.integer(limit)
|
||||
if (length(limit) != 1L || is.na(limit) || limit < 0L) {
|
||||
cli::cli_abort("`limit` must be a single non-negative integer.",
|
||||
class = "uscogdata_invalid_pagination")
|
||||
}
|
||||
offset <- if (is.null(offset)) 0L else as.integer(offset)
|
||||
if (length(offset) != 1L || is.na(offset) || offset < 0L) {
|
||||
cli::cli_abort("`offset` must be a single non-negative integer.",
|
||||
class = "uscogdata_invalid_pagination")
|
||||
}
|
||||
if (complete) {
|
||||
cli::cli_abort(c(
|
||||
"`limit`/`offset` cannot be combined with `complete = TRUE`.",
|
||||
@@ -466,24 +459,14 @@ cog_spending <- function(govid = NULL, years, category = NULL,
|
||||
limit = limit, offset = offset)
|
||||
result <- tibble::as_tibble(DBI::dbGetQuery(con, sql))
|
||||
if (!is.null(limit)) {
|
||||
# COUNT(*) OVER() rides along as an ordinary column so the total comes
|
||||
# from the same scan when this page has any rows -- see
|
||||
# .build_verb_sql(). An empty page (offset past the end) carries no
|
||||
# such row to read it from, so that one case falls back to a second,
|
||||
# unpaginated COUNT(*) query rather than reporting a wrong zero.
|
||||
if (nrow(result) > 0L) {
|
||||
total_rows <- result$pagination_total_rows[[1]]
|
||||
result$pagination_total_rows <- NULL
|
||||
} else {
|
||||
count_sql <- sprintf(
|
||||
"SELECT COUNT(*) AS n FROM (%s) AS _uncounted",
|
||||
.build_verb_sql(view, subtype_col, cohort, years,
|
||||
if (all_categories) NULL else category,
|
||||
ig_view, subtype_scope,
|
||||
all_categories = all_categories)
|
||||
)
|
||||
total_rows <- as.integer(DBI::dbGetQuery(con, count_sql)$n[[1]])
|
||||
}
|
||||
paged <- .take_pagination_total(result, con, function() {
|
||||
.build_verb_sql(view, subtype_col, cohort, years,
|
||||
if (all_categories) NULL else category,
|
||||
ig_view, subtype_scope,
|
||||
all_categories = all_categories)
|
||||
})
|
||||
result <- paged$result
|
||||
total_rows <- paged$total_rows
|
||||
}
|
||||
}
|
||||
|
||||
@@ -873,22 +856,9 @@ cog_spending <- function(govid = NULL, years, category = NULL,
|
||||
# matching row across the network only to slice and discard most of it
|
||||
# afterward (the pattern behind the 2026-08-06 production incident: a
|
||||
# 193,105-row/194-page sweep re-ran the full query and re-listified every
|
||||
# row on EVERY page). COUNT(*) OVER() rides along as an ordinary column so
|
||||
# the caller gets the true total from this same scan -- see the call site
|
||||
# in .verb_spendrev(), which reads it off row 1 and strips it back out.
|
||||
# The outer SELECT * wrapping (rather than appending LIMIT/OFFSET directly
|
||||
# to base_sql) is what makes COUNT(*) OVER() see the post-GROUP-BY row
|
||||
# count, not the pre-aggregation one.
|
||||
if (is.null(limit)) {
|
||||
base_sql
|
||||
} else {
|
||||
sprintf(
|
||||
"SELECT *, COUNT(*) OVER() AS pagination_total_rows
|
||||
FROM (%s) AS _paged
|
||||
LIMIT %d OFFSET %d",
|
||||
base_sql, limit, offset
|
||||
)
|
||||
}
|
||||
# row on EVERY page). See .paginate_sql() in R/pagination.R for why the
|
||||
# wrapping is an outer SELECT rather than a bare LIMIT on base_sql.
|
||||
.paginate_sql(base_sql, limit, offset)
|
||||
}
|
||||
|
||||
#' Join population onto a result and derive the per-capita columns.
|
||||
|
||||
Reference in New Issue
Block a user