diff --git a/DESCRIPTION b/DESCRIPTION index d380818..f91acc5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,7 +14,6 @@ LazyData: false Depends: R (>= 4.1) Imports: DBI (>= 1.1.0), - dbplyr (>= 2.4.0), duckdb (>= 1.0.0), dplyr (>= 1.1.0), tibble, diff --git a/NAMESPACE b/NAMESPACE index 5379967..b0f48df 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,8 @@ export(cog_explain) export(cog_find_peers) export(cog_geographic_rollup) +export(cog_gov_search) +export(cog_mirror) export(cog_peer_compare) export(cog_revenue) export(cog_spending) diff --git a/R/mirror.R b/R/mirror.R new file mode 100644 index 0000000..655733c --- /dev/null +++ b/R/mirror.R @@ -0,0 +1,140 @@ +# R/mirror.R + +#' Mirror the published corpus to a local directory +#' +#' Downloads (or copies, for local-path fixture URLs) every file listed in +#' the session's `manifest.json` into `dest`, preserving the relative path +#' structure. Files already present with a matching SHA-256 hash are +#' skipped unless `overwrite = TRUE`. The resulting directory can be +#' pointed at via `options(uscogdata.url = paste0(dest, "/"))` for offline +#' analysis. +#' +#' @param dest Destination directory. Created if missing. +#' @param include Subset of `c("long", "metadata", "docs")` controlling +#' which manifest sections to mirror. +#' @param overwrite If `TRUE`, re-copy even when the local SHA matches. +#' @param progress If `TRUE`, show a cli progress bar. +#' @return Invisibly, a tibble of `(path, sha256, size_bytes, status)` +#' rows where `status` is `"downloaded"` or `"cached"`. +#' @export +cog_mirror <- function(dest, + include = c("long", "metadata", "docs"), + overwrite = FALSE, + progress = interactive()) { + if (!is.character(dest) || length(dest) != 1L) { + cli::cli_abort("`dest` must be a length-1 character path.") + } + include <- match.arg(include, several.ok = TRUE) + if (!dir.exists(dest)) dir.create(dest, recursive = TRUE) + + .ensure_session() + url <- .uscogdata_env$url + manifest <- .uscogdata_env$manifest + is_local <- .is_local_path(url) + + .write_manifest_local(manifest, file.path(dest, "manifest.json")) + + entries <- .collect_mirror_entries(manifest, include) + results <- vector("list", length(entries)) + pb <- if (isTRUE(progress)) { + cli::cli_progress_bar("Mirroring", total = length(entries)) + } else NULL + + for (i in seq_along(entries)) { + e <- entries[[i]] + dest_path <- file.path(dest, e$path) + .ensure_parent_dir(dest_path) + status <- .mirror_one_file( + src_url = paste0(url, e$path), + dest_path = dest_path, + expected_sha = e$sha256, + is_local = is_local, + overwrite = overwrite + ) + results[[i]] <- tibble::tibble( + path = e$path, + sha256 = e$sha256 %||% NA_character_, + size_bytes = as.integer(file.info(dest_path)$size), + status = status + ) + if (!is.null(pb)) cli::cli_progress_update() + } + if ("docs" %in% include) { + .mirror_docs(url, dest, is_local, overwrite) + } + if (!is.null(pb)) cli::cli_progress_done() + invisible(dplyr::bind_rows(results)) +} + +#' @noRd +.write_manifest_local <- function(manifest, path) { + writeLines( + jsonlite::toJSON(manifest, auto_unbox = TRUE, pretty = TRUE), + path + ) +} + +#' @noRd +.collect_mirror_entries <- function(manifest, include) { + entries <- list() + if ("long" %in% include && length(manifest$files$long_partitions) > 0L) { + entries <- c(entries, manifest$files$long_partitions) + } + if ("metadata" %in% include && length(manifest$files$metadata) > 0L) { + entries <- c(entries, manifest$files$metadata) + } + entries +} + +#' @noRd +.ensure_parent_dir <- function(path) { + d <- dirname(path) + if (!dir.exists(d)) dir.create(d, recursive = TRUE) +} + +#' @noRd +.mirror_one_file <- function(src_url, dest_path, expected_sha, + is_local, overwrite) { + if (!isTRUE(overwrite) && file.exists(dest_path) && + !is.null(expected_sha) && + digest::digest(dest_path, algo = "sha256", file = TRUE) == expected_sha) { + return("cached") + } + if (isTRUE(is_local)) { + # src_url was built as paste0(url, e$path); url ends in "/" + src <- src_url + if (!file.exists(src)) { + cli::cli_abort("Source file missing: {src}") + } + ok <- file.copy(src, dest_path, overwrite = TRUE) + if (!isTRUE(ok)) cli::cli_abort("file.copy failed: {src} -> {dest_path}") + } else { + resp <- httr2::request(src_url) |> + httr2::req_error(is_error = function(r) httr2::resp_status(r) >= 400) |> + httr2::req_perform() + writeBin(httr2::resp_body_raw(resp), dest_path) + } + "downloaded" +} + +#' @noRd +.mirror_docs <- function(url, dest, is_local, overwrite) { + docs <- c("README.md", "data_dictionary.md", + "series_breaks.md", "reader-specification.md") + for (doc in docs) { + src <- paste0(url, "docs/", doc) + dst <- file.path(dest, "docs", doc) + .ensure_parent_dir(dst) + if (isTRUE(is_local)) { + if (file.exists(src)) file.copy(src, dst, overwrite = TRUE) + # silently skip missing optional docs in local fixtures + next + } + try({ + resp <- httr2::request(src) |> + httr2::req_error(is_error = function(r) httr2::resp_status(r) >= 400) |> + httr2::req_perform() + writeBin(httr2::resp_body_raw(resp), dst) + }, silent = TRUE) + } +} diff --git a/R/search.R b/R/search.R new file mode 100644 index 0000000..01557b1 --- /dev/null +++ b/R/search.R @@ -0,0 +1,122 @@ +# R/search.R + +#' Search for governments by name, state, and/or type +#' +#' Returns rows from `canonical_fips_xwalk` matching the supplied filters. +#' Intended as the entry point users call to resolve a human-readable place +#' name into one or more `canonical_govid` values before calling +#' [cog_spending()] / [cog_revenue()] / etc. +#' +#' @param pattern Character regex matched case-insensitively against +#' `gov_name`. `NULL` (default) means no name filter. +#' @param state Either a 2-letter USPS abbreviation (e.g. `"FL"`), a FIPS +#' integer (e.g. `12`), or `NULL`. +#' @param type Government type: an integer in `0:3` or one of `"state"`, +#' `"county"`, `"city"`, `"township"`. Passing `4`, `5`, +#' `"special_district"`, or `"school_district"` emits an explanatory +#' message and returns an empty tibble (v0.1 corpus excludes those types). +#' @return Tibble from `canonical_fips_xwalk` sorted by `population_acs` +#' descending (`NULL`s last). +#' @export +cog_gov_search <- function(pattern = NULL, state = NULL, type = NULL) { + if (!is.null(type) && .is_excluded_type(type)) { + cli::cli_inform(c( + i = "v0.1 covers gov_types 0-3 (state/county/city/township) only.", + i = "Types 4 (special districts) and 5 (school districts) are excluded; see vignette('coverage-scope')." + )) + return(.empty_xwalk_tibble()) + } + con <- .ensure_session() + + preds <- character(0) + if (!is.null(pattern)) { + if (!is.character(pattern) || length(pattern) != 1L) { + cli::cli_abort("`pattern` must be a length-1 character string.") + } + preds <- c(preds, + sprintf("regexp_matches(gov_name, %s, 'i')", + .sql_lit_chr(pattern))) + } + if (!is.null(state)) { + st_fips <- .coerce_state_to_fips(state) + preds <- c(preds, sprintf("fips_state = %s", .sql_lit_chr(st_fips))) + } + if (!is.null(type)) { + int_type <- .coerce_type(type) + preds <- c(preds, sprintf("govs_type = %d", int_type)) + } + + where <- if (length(preds) == 0L) "" else paste("WHERE", paste(preds, collapse = " AND ")) + sql <- paste( + "SELECT * FROM canonical_fips_xwalk", + where, + "ORDER BY population_acs DESC NULLS LAST" + ) + tibble::as_tibble(DBI::dbGetQuery(con, sql)) +} + +#' @noRd +.empty_xwalk_tibble <- function() { + tibble::tibble( + canonical_govid = character(0), gov_name = character(0), + govs_type = integer(0), type_label = character(0), + fips_state = character(0), fips_county = character(0), + fips_place = character(0), first_year = integer(0), + last_year = integer(0), population_acs = integer(0), + confidence = character(0) + ) +} + +#' @noRd +.is_excluded_type <- function(type) { + excluded <- c("4", "5", "special_district", "school_district") + as.character(type) %in% excluded +} + +#' @noRd +.coerce_type <- function(type) { + if (is.numeric(type) || + (is.character(type) && length(type) == 1L && grepl("^[0-9]+$", type))) { + n <- as.integer(type) + if (!n %in% 0:3) { + cli::cli_abort("type must be 0, 1, 2, or 3 (v0.1 scope).") + } + return(n) + } + map <- c(state = 0L, county = 1L, city = 2L, township = 3L) + key <- as.character(type) + if (!key %in% names(map)) cli::cli_abort("Unknown type: {type}.") + map[[key]] +} + +#' @noRd +.coerce_state_to_fips <- function(state) { + if (is.numeric(state) || + (is.character(state) && length(state) == 1L && grepl("^[0-9]+$", state))) { + return(sprintf("%02d", as.integer(state))) + } + if (!is.character(state) || length(state) != 1L) { + cli::cli_abort("`state` must be a 2-letter USPS abbrev or a FIPS integer.") + } + fips <- .state_abbrev_to_fips[[toupper(state)]] + if (is.null(fips)) { + cli::cli_abort("Unknown state abbreviation: {state}.") + } + fips +} + +# USPS state / territory abbreviation -> 2-digit FIPS code. +# Includes 50 states + DC + territories. Note FIPS 66 = GU (not GA). +#' @noRd +.state_abbrev_to_fips <- c( + AL = "01", AK = "02", AZ = "04", AR = "05", CA = "06", CO = "08", + CT = "09", DE = "10", DC = "11", FL = "12", GA = "13", HI = "15", + ID = "16", IL = "17", IN = "18", IA = "19", KS = "20", KY = "21", + LA = "22", ME = "23", MD = "24", MA = "25", MI = "26", MN = "27", + MS = "28", MO = "29", MT = "30", NE = "31", NV = "32", NH = "33", + NJ = "34", NM = "35", NY = "36", NC = "37", ND = "38", OH = "39", + OK = "40", OR = "41", PA = "42", RI = "44", SC = "45", SD = "46", + TN = "47", TX = "48", UT = "49", VT = "50", VA = "51", WA = "53", + WV = "54", WI = "55", WY = "56", + AS = "60", GU = "66", MP = "69", PR = "72", VI = "78" +) diff --git a/R/session.R b/R/session.R index 4f5438c..a85365b 100644 --- a/R/session.R +++ b/R/session.R @@ -33,6 +33,34 @@ cog_open <- function(url = .resolve_url(), .uscogdata_env$con } +#' @noRd +#' Check which of the supplied govids exist in canonical_fips_xwalk. +#' Emits a cli message listing any missing ones alongside a pointer to the +#' v0.1 scope explanation; returns both sets so callers can attach them to +#' provenance. +.check_govids_in_scope <- function(govids) { + govids <- unique(as.character(govids)) + if (length(govids) == 0L) return(list(found = character(0), missing = character(0))) + con <- .ensure_session() + sql <- sprintf( + "SELECT canonical_govid FROM canonical_fips_xwalk WHERE canonical_govid IN (%s)", + .sql_lit_chr(govids) + ) + found <- DBI::dbGetQuery(con, sql)$canonical_govid + missing <- setdiff(govids, found) + if (length(missing) > 0L) { + n <- length(missing) + shown <- paste(utils::head(missing, 5L), collapse = ", ") + more <- if (n > 5L) sprintf(" (+%d more)", n - 5L) else "" + cli::cli_inform(c( + i = sprintf("%d govid%s not found in v0.1 corpus: %s%s", + n, if (n == 1L) "" else "s", shown, more), + i = "v0.1 covers gov_types 0-3 (state/county/city/township). Types 4/5 excluded; see vignette('coverage-scope')." + )) + } + list(found = found, missing = missing) +} + #' @noRd cog_close <- function() { if (!is.null(.uscogdata_env$con) && DBI::dbIsValid(.uscogdata_env$con)) { diff --git a/R/spending.R b/R/spending.R index 4cb5dc4..680c6f6 100644 --- a/R/spending.R +++ b/R/spending.R @@ -49,6 +49,7 @@ cog_spending <- function(govid, years, category = NULL, if (!is.null(adjust_to_year)) adjust_to_year <- as.integer(adjust_to_year) con <- .ensure_session() + scope <- .check_govids_in_scope(govid) sql <- .build_verb_sql(view, subtype_col, govid, years, category) result <- tibble::as_tibble(DBI::dbGetQuery(con, sql)) @@ -60,7 +61,7 @@ cog_spending <- function(govid, years, category = NULL, result$notes <- .notes_column(result) - attr(result, "provenance") <- .build_provenance( + prov <- .build_provenance( verb = verb, call = call, govid = govid, @@ -72,6 +73,9 @@ cog_spending <- function(govid, years, category = NULL, sql = sql, subtype_col = subtype_col ) + prov$scope$govids_found <- scope$found + prov$scope$govids_missing <- scope$missing + attr(result, "provenance") <- prov result } diff --git a/man/cog_gov_search.Rd b/man/cog_gov_search.Rd new file mode 100644 index 0000000..05f24ee --- /dev/null +++ b/man/cog_gov_search.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/search.R +\name{cog_gov_search} +\alias{cog_gov_search} +\title{Search for governments by name, state, and/or type} +\usage{ +cog_gov_search(pattern = NULL, state = NULL, type = NULL) +} +\arguments{ +\item{pattern}{Character regex matched case-insensitively against +`gov_name`. `NULL` (default) means no name filter.} + +\item{state}{Either a 2-letter USPS abbreviation (e.g. `"FL"`), a FIPS +integer (e.g. `12`), or `NULL`.} + +\item{type}{Government type: an integer in `0:3` or one of `"state"`, +`"county"`, `"city"`, `"township"`. Passing `4`, `5`, +`"special_district"`, or `"school_district"` emits an explanatory +message and returns an empty tibble (v0.1 corpus excludes those types).} +} +\value{ +Tibble from `canonical_fips_xwalk` sorted by `population_acs` + descending (`NULL`s last). +} +\description{ +Returns rows from `canonical_fips_xwalk` matching the supplied filters. +Intended as the entry point users call to resolve a human-readable place +name into one or more `canonical_govid` values before calling +[cog_spending()] / [cog_revenue()] / etc. +} diff --git a/man/cog_mirror.Rd b/man/cog_mirror.Rd new file mode 100644 index 0000000..23b292e --- /dev/null +++ b/man/cog_mirror.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/mirror.R +\name{cog_mirror} +\alias{cog_mirror} +\title{Mirror the published corpus to a local directory} +\usage{ +cog_mirror( + dest, + include = c("long", "metadata", "docs"), + overwrite = FALSE, + progress = interactive() +) +} +\arguments{ +\item{dest}{Destination directory. Created if missing.} + +\item{include}{Subset of `c("long", "metadata", "docs")` controlling +which manifest sections to mirror.} + +\item{overwrite}{If `TRUE`, re-copy even when the local SHA matches.} + +\item{progress}{If `TRUE`, show a cli progress bar.} +} +\value{ +Invisibly, a tibble of `(path, sha256, size_bytes, status)` + rows where `status` is `"downloaded"` or `"cached"`. +} +\description{ +Downloads (or copies, for local-path fixture URLs) every file listed in +the session's `manifest.json` into `dest`, preserving the relative path +structure. Files already present with a matching SHA-256 hash are +skipped unless `overwrite = TRUE`. The resulting directory can be +pointed at via `options(uscogdata.url = paste0(dest, "/"))` for offline +analysis. +} diff --git a/tests/testthat/test-mirror.R b/tests/testthat/test-mirror.R new file mode 100644 index 0000000..6e812cb --- /dev/null +++ b/tests/testthat/test-mirror.R @@ -0,0 +1,67 @@ +test_that("cog_mirror copies manifest + metadata files", { + skip_if_no_corpus() + tmp <- tempfile("uscogmirror_"); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE)) + + r <- cog_mirror(tmp, include = "metadata", progress = FALSE) + expect_s3_class(r, "tbl_df") + expect_true(file.exists(file.path(tmp, "manifest.json"))) + expect_true(file.exists(file.path(tmp, "data/canonical_fips_xwalk.parquet"))) + expect_true(file.exists(file.path(tmp, "data/summary_categories.parquet"))) + expect_true(all(r$status == "downloaded")) + expect_true(all(c("path", "sha256", "size_bytes", "status") %in% names(r))) +}) + +test_that("cog_mirror is idempotent when SHA matches (status = 'cached')", { + skip_if_no_corpus() + tmp <- tempfile("uscogmirror_"); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE)) + + cog_mirror(tmp, include = "metadata", progress = FALSE) + r2 <- cog_mirror(tmp, include = "metadata", progress = FALSE) + expect_true(all(r2$status == "cached")) +}) + +test_that("cog_mirror overwrite = TRUE re-copies even when SHA matches", { + skip_if_no_corpus() + tmp <- tempfile("uscogmirror_"); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE)) + + cog_mirror(tmp, include = "metadata", progress = FALSE) + r3 <- cog_mirror(tmp, include = "metadata", progress = FALSE, overwrite = TRUE) + expect_true(all(r3$status == "downloaded")) +}) + +test_that("cog_mirror verifies sha256 against manifest", { + skip_if_no_corpus() + tmp <- tempfile("uscogmirror_"); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE)) + + r <- cog_mirror(tmp, include = "metadata", progress = FALSE) + for (i in seq_len(nrow(r))) { + got <- digest::digest(file.path(tmp, r$path[i]), + algo = "sha256", file = TRUE) + expect_equal(got, r$sha256[i]) + } +}) + +test_that("cog_mirror reads back via a fresh session against the mirror", { + skip_if_no_corpus() + tmp <- tempfile("uscogmirror_"); dir.create(tmp) + on.exit(unlink(tmp, recursive = TRUE)) + + cog_mirror(tmp, include = c("long", "metadata"), progress = FALSE) + + # Re-open against the local mirror and run a query. + old_url <- getOption("uscogdata.url") + on.exit({ + cog_close() + options(uscogdata.url = old_url) + }, add = TRUE) + cog_close() + options(uscogdata.url = paste0(normalizePath(tmp), "/")) + + r <- cog_spending("101006006", 2020L, "Corrections") + expect_gt(nrow(r), 0L) + expect_equal(unique(r$canonical_govid), "101006006") +}) diff --git a/tests/testthat/test-search.R b/tests/testthat/test-search.R new file mode 100644 index 0000000..ddc3ffa --- /dev/null +++ b/tests/testthat/test-search.R @@ -0,0 +1,74 @@ +test_that("cog_gov_search by name pattern returns matches", { + skip_if_no_corpus() + r <- cog_gov_search("^BROWARD") + expect_s3_class(r, "tbl_df") + expect_true(all(grepl("^BROWARD", r$gov_name))) + expect_true("canonical_govid" %in% names(r)) +}) + +test_that("cog_gov_search case-insensitive", { + skip_if_no_corpus() + r <- cog_gov_search("broward") + expect_gt(nrow(r), 0L) +}) + +test_that("cog_gov_search by state abbreviation", { + skip_if_no_corpus() + r <- cog_gov_search(state = "FL", type = 1L) + expect_true(all(r$fips_state == "12")) + expect_true(all(r$govs_type == 1L)) +}) + +test_that("cog_gov_search accepts FIPS int for state", { + skip_if_no_corpus() + r <- cog_gov_search(state = 12, type = "county") + expect_true(all(r$fips_state == "12")) + expect_true(all(r$govs_type == 1L)) +}) + +test_that("cog_gov_search accepts type as name string", { + skip_if_no_corpus() + for (pair in list(c("state", 0), c("county", 1), c("city", 2), c("township", 3))) { + r <- cog_gov_search(type = pair[1]) + expect_true(all(r$govs_type == as.integer(pair[2]))) + } +}) + +test_that("cog_gov_search for excluded types emits message and returns empty", { + skip_if_no_corpus() + expect_message( + r <- cog_gov_search(type = 4L), + "v0.1|excluded" + ) + expect_equal(nrow(r), 0L) + expect_true("canonical_govid" %in% names(r)) + expect_message( + r2 <- cog_gov_search(type = "special_district"), + "v0.1|excluded" + ) + expect_equal(nrow(r2), 0L) +}) + +test_that("cog_gov_search combining filters", { + skip_if_no_corpus() + r <- cog_gov_search("BROWARD", state = "FL") + expect_true(all(grepl("BROWARD", r$gov_name))) + expect_true(all(r$fips_state == "12")) +}) + +test_that("cog_gov_search sorts by population desc", { + skip_if_no_corpus() + r <- cog_gov_search(state = "FL", type = 2L) + nonNA <- r$population_acs[!is.na(r$population_acs)] + expect_true(all(diff(nonNA) <= 0)) +}) + +test_that("cog_gov_search rejects unknown type string", { + expect_error(cog_gov_search(type = "galaxy"), "Unknown type") +}) + +test_that("cog_gov_search with no filters returns full registry", { + skip_if_no_corpus() + r <- cog_gov_search() + expect_gt(nrow(r), 1000L) +}) diff --git a/tests/testthat/test-spending.R b/tests/testthat/test-spending.R index 80c72dd..a83b182 100644 --- a/tests/testthat/test-spending.R +++ b/tests/testthat/test-spending.R @@ -50,14 +50,29 @@ test_that("cog_spending with per_capita + adjust_to_year adds all columns", { names(r))) }) -test_that("cog_spending for unknown govid returns empty tibble", { +test_that("cog_spending for unknown govid returns empty tibble + informs", { skip_if_no_corpus() - r <- cog_spending("XXXINVALID", 2020L, "Corrections") + expect_message( + r <- cog_spending("XXXINVALID", 2020L, "Corrections"), + "not found|v0.1" + ) expect_s3_class(r, "tbl_df") expect_equal(nrow(r), 0L) expect_true("notes" %in% names(r)) - # provenance still attached - expect_false(is.null(attr(r, "provenance"))) + prov <- attr(r, "provenance") + expect_false(is.null(prov)) + expect_equal(prov$scope$govids_missing, "XXXINVALID") + expect_equal(length(prov$scope$govids_found), 0L) +}) + +test_that("cog_spending records found + missing govids in provenance", { + skip_if_no_corpus() + suppressMessages( + r <- cog_spending(c("101006006", "XXXINVALID"), 2020L, "Corrections") + ) + prov <- attr(r, "provenance") + expect_equal(sort(prov$scope$govids_found), "101006006") + expect_equal(sort(prov$scope$govids_missing), "XXXINVALID") }) test_that("cog_spending result has provenance attribute matching schema", {