diff --git a/.Rbuildignore b/.Rbuildignore index 7205877..ba6315b 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,17 +3,17 @@ ^\.Rproj\.user$ ^_pkgdown\.yml$ ^docs$ +^Meta$ +^doc$ ^pkgdown$ ^\.github$ ^LICENSE\.md$ ^\.git$ ^\.gitignore$ \.gitkeep$ -^vignettes$ ^specs$ ^plans$ -^doc$ -^Meta$ ^\.gitea$ ^CLAUDE\.md$ ^\.superpowers$ +^CONTRIBUTING\.md$ diff --git a/CLAUDE.md b/CLAUDE.md index 5fdb610..81acc45 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,3 +124,10 @@ devtools::test() a numbered `.sql` file; build a query in R. - No arrow dependency — DuckDB reads parquet natively - `withr` is a Suggests-only dep; only used in tests + +## Domain context — read this first + +**Before doing any work in this repo, read `~/.claude/memory/values/civilytics.md`.** +It carries the purpose, direction, and constraints for this domain. It is not optional +context — read it before planning or writing code, not after. (An `@` import will not +work here; project-level imports don't preload. The read is the mechanism.) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f296d70 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,105 @@ +# Contributing to uscogdata + +Thanks for reading this — a package like this gets better mostly through people +noticing that a number looks wrong. + +## Where the code lives + +Development happens on **Gitea**, at +`gitea.civilytics.org/Civilytics/uscogdata`. The repository at +`github.com/civilytics/uscogdata` is a **mirror** that accepts issues and pull +requests. + +## What happens to a GitHub pull request + +Open it normally. Behind the scenes it is fetched and landed on the canonical +Gitea repository, then syncs back: + +```sh +git fetch github refs/pull/42/head:pr-42 +git switch main && git merge --no-ff pr-42 +git push origin main # Gitea -> mirror -> GitHub +``` + +Because the merge preserves your commits at their original SHAs, **GitHub marks +your PR merged on its own** as soon as the mirror syncs. So: + +> If your pull request closes as "Merged" without anyone visibly clicking +> Merge, that is the normal, successful outcome — not a rejection. + +Substantial contributions get a `ctb` entry in `DESCRIPTION`, which surfaces in +`citation("uscogdata")`. + +There is no CLA and no DCO sign-off requirement. + +## Running the tests + +```r +devtools::test() # bundled fixture; no network, no credentials +``` + +`tests/testthat/setup.R` points `USCOGDATA_URL` at +`inst/extdata/fixture_corpus/` automatically — a four-year slice (2011, 2012, +2019, 2020) covering all 50 states. That is the whole data setup. + +## Testing against the live corpus + +```sh +USCOGDATA_LIVE_TEST=true Rscript -e 'devtools::test(filter = "live-corpus")' +``` + +This is worth understanding rather than skipping. Until 0.3.0 the package +**could not read a remote corpus at all** — the partitioned view used a glob, +and DuckDB cannot expand a glob over generic HTTP. It went unnoticed for months +because every test path used a local corpus (the bundled fixture), and so did +the production API (a host mount). Nothing exercised the package the way a new +user does. + +`test-live-corpus.R` is the only test that runs with no `USCOGDATA_URL`, no +option, and no fixture. If you change anything touching view registration, +manifest handling, or configuration, run it. + +## Do not exclude the fixture from the build + +There is a temptation to add `^inst/extdata/fixture_corpus$` to +`.Rbuildignore` because 15 MB feels large for a package. Don't: + +- `vignette("total-spending")` reads from it and would fail to build. +- `R CMD check` on r-universe and GitHub Actions would have no corpus, so the + suite could not run without credentials. + +This package is not going to CRAN, so its 5 MB guidance does not apply. A +package-size NOTE in `R CMD check` is expected and acceptable. + +## Downstream consumers + +`cog-api` depends on this package and its CI clones uscogdata at +`USCOGDATA_REF`, **defaulting to `main`**. There is no pin. Anything merged +here reaches the API's next build, so before merging a change to the reader, +run the API suite against your branch: + +```sh +Rscript -e "remotes::install_local('/path/to/uscogdata', upgrade = 'never')" +cd /path/to/cog-api/api/tests/testthat +Rscript -e 'testthat::test_dir(".", stop_on_failure = TRUE)' +``` + +The API calls only exported verbs, so internal refactors are usually safe — +but "usually" is not a release gate. + +## Release checklist + +1. `devtools::test()` — green against the bundled fixture, offline. +2. `USCOGDATA_LIVE_TEST=true devtools::test()` — green against the live corpus. +3. cog-api suite green against this branch (above). +4. `devtools::check(args = "--as-cran")` — 0 errors, 0 warnings. +5. `pkgdown::build_site()` completes. +6. Vignettes resolve from an installed copy: + `vignette("total-spending", package = "uscogdata")`. +7. **Cold-start check**: on a machine that has never had this package, + install it and run the README quickstart verbatim with no environment + variables set. This is the only check that catches a + corpus-unreachable defect, and its absence is why 0.3.0 needed fixing. +8. Bump `Version` and add a `NEWS.md` section. +9. Tag, then update the r-universe registry pin at + `github.com/civilytics/civilytics.r-universe.dev`. diff --git a/DESCRIPTION b/DESCRIPTION index 8ccdcc3..7159bab 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,14 +1,21 @@ Package: uscogdata Type: Package Title: Curated Reader for the Civilytics US Census of Governments Finance Corpus -Version: 0.2.0 -Authors@R: - person("Civilytics", , , "jknowles@gmail.com", role = c("aut", "cre")) +Version: 0.3.0 +Authors@R: c( + person(c("Jared", "E."), "Knowles", + email = "jared@civilytics.com", + role = c("aut", "cre"), + comment = c(ORCID = "0000-0003-0005-9478")), + person("Civilytics Consulting LLC", role = c("cph", "fnd"))) Description: Curated R verbs over the Civilytics US Census of Governments finance corpus. Provides unit-level financial profiles, geographic rollups, and peer comparisons with auditable provenance and built-in cross-vintage correctness. License: MIT + file LICENSE +URL: https://github.com/civilytics/uscogdata, + https://civilytics.r-universe.dev/uscogdata +BugReports: https://github.com/civilytics/uscogdata/issues Encoding: UTF-8 LazyData: false Depends: R (>= 4.1) @@ -32,4 +39,4 @@ Config/testthat/edition: 3 VignetteBuilder: knitr RoxygenNote: 7.3.3 MinCorpusSchema: 4 -MaxCorpusSchema: 5 +MaxCorpusSchema: 7 diff --git a/LICENSE b/LICENSE index 3bb8b6e..488811e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,2 +1,2 @@ YEAR: 2026 -COPYRIGHT HOLDER: Civilytics +COPYRIGHT HOLDER: Civilytics Consulting LLC diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..aa9e229 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2026 Civilytics Consulting LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NEWS.md b/NEWS.md index 0b43b3f..cace5cd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,63 @@ +# uscogdata 0.3.0 + +First public release. + +`uscogdata` provides curated R verbs over the Civilytics US Census of +Governments finance corpus: unit-level financial profiles, geographic rollups +and peer comparisons, with auditable provenance on every result. + +## What it covers + +Government types 0-3 (state, county, municipality, township), FY1967-FY2024 -- +56 fiscal years, 46,148,034 rows, 190.6 MB. There is no source data for FY1968 +or FY1969. Special districts (type 4) and school districts (type 5) are out of +scope pending validation. + +## The verbs + +`cog_spending()`, `cog_revenue()` and `cog_balances()` for flows and holdings; +`cog_gov_search()` to resolve place names (including basket mode for many at +once); `cog_find_peers()` and `cog_peer_compare()` for cohorts; +`cog_geographic_rollup()` for aggregates; `cog_categories()`, `cog_recipes()`, +`cog_manifest()` and `cog_explain()` for metadata and provenance; and +`cog_mirror()` for a local copy of the corpus. + +## Reading the corpus now works out of the box + +* The package reads the published corpus over HTTPS **with no configuration**. + Previously the default was a placeholder sentinel and no document in the + package supplied a working URL, so a new user had no path to a session. +* Remote reads work at all. The partitioned view used a glob, and DuckDB + cannot expand a glob over generic HTTP -- there is no directory listing to + expand against. Partition paths are now enumerated from the corpus manifest, + which is host-agnostic: an HTTPS mirror, a Nextcloud share and a local + `cog_mirror()` copy all take the same path. +* Nothing is written to disk in remote mode; DuckDB fetches only the row + groups a query needs. + +## Four things to know before your first query + +* **Amounts are in full US dollars.** The raw Census files report thousands; + the verbs multiply by 1000 on the way out. Do not multiply again. +* **Multi-government aggregates disclose their coverage.** The Census is a + complete enumeration only in years ending in 2 and 7; every other year is a + sample. Every such result carries `provenance$coverage` with per-year + `n_units_reporting`. +* **Absence means two different things.** Before FY2012 an absent cell means + Census published $0; from FY2012 it means not reported. `complete = TRUE` + labels which. +* **Series breaks reach you unasked.** Catalogued breaks intersecting your + query appear in provenance and in `cog_explain()`. + +## Known limits + +* Special districts (type 4) and school districts (type 5) are out of scope. +* Per-capita rollups exclude governments with no F-33 population, which is by + design but does silently narrow a rollup. +* `n_units_reporting` is category-conditional and is not a response rate. +* Employee-retirement (`X`) codes stop at FY2016, when those systems moved to + the Annual Survey of Public Pensions. + # uscogdata 0.2.0 ## New features @@ -39,262 +99,3 @@ is not a response rate: a government that was surveyed and genuinely spends nothing in the requested category is indistinguishable from one never surveyed (uscogdata#36). - -# uscogdata 0.1.0 (development) - -## Signposting now catches partially-suppressed categories - -* A coverage suggestion used to fire only when a category returned **no rows - at all** in a requested year. That missed the more dangerous case: a - category that still returns rows while silently dropping component codes - the wide era publishes only as aggregates (#9). `cog_spending(category = - "Public Welfare")` for FY2011 returned a plausible figure that omitted - `E67`/`E68` entirely -- for Los Angeles County, $2,075,461,000 of a true - $5,261,404,000, a 39% understatement, with `provenance$suggestions` empty. -* Suggestions now also fire on **partial** coverage, and every suggestion - carries `trigger` (`"empty_year"` or `"suppressed_component"`), - `suppressed_amount`, `suppressed_years` and `suppressed_codes`, so a caller - can see how much is missing and decide whether to re-run with the recipe. -* `cog_revenue()` gets the same fix through the shared verb path. Alaska's - FY2011 `Miscellaneous Revenue` reported $943,842,000 while dropping - $1,899,995,000 of aggregate-published `U4-` rents and royalties. -* The trigger stays recipe-driven, so it only fires where a harmonization - recipe actually exists to name the fix. `higher_ed_e18_wide` and - `general_gov_e89_wide` stay silent in every year measured on the bundled - fixture, because their components are ordinary classified leaves even - pre-2012. -* The `suppressed_component` trigger (and any `suppressed_amount`/ - `suppressed_codes` an `empty_year` fire also carries) is scoped to the - calling verb's own flow family: `cog_spending()` only ever measures E/F/G - component dollars, `cog_revenue()` only T/A/U/B/C/D. A component from the - OTHER flow family reports `suppressed_amount = 0` rather than a fabricated - claim. The `empty_year` trigger itself is not flow-scoped -- a category - belonging to the other flow (e.g. `cog_spending(category = "IG Local")`) - still returns zero rows and can still fire, in any year including modern - ones, naming the recipe whose own generic join finds real data for this - government. That is a mis-scoped query, not a corpus-format gap, so its - `suppressed_amount` is correctly 0. - -## New: `cog_balances()` for cash-and-security holdings - -* New `cog_balances()` exposes the 14 cash-and-security holding codes - (`category_type = "balance"`): fund balances, retirement system holdings and - insurance trust balances (#25). Holdings are a stock, not a flow, so the verb - has no `expenditure_concept` / `revenue_concept` / `complete` arguments, and - no `subtype` argument either -- for holdings, `category` is a strict - coarsening of `balance_subtype`, so `category = "Fund Balances"` is exactly - the `general` family (`W01`/`W31`/`W61`). -* `cog_balances()` results carry `provenance$balance_caveats`, recording that - Census holdings are gross rather than GAAP fund balance, and the measured - coverage window of each subtype family. - -## Multi-government aggregates now disclose their reporting coverage - -* The Census of Governments is a **complete census only in years ending in 2 - and 7**; every other year is a sample, and the sample varies enormously. On - the bundled fixture, Wisconsin's 608-city universe rolls up **597** - governments in FY2012 and **112** in FY2019 — an 18%-to-98% swing the - return value said nothing about, so a statewide total resting on a fifth of - the universe looked exactly like one resting on all of it. -* `cog_geographic_rollup()`, `cog_peer_compare()` and `cog_find_peers()` gain - `coverage`: - - | value | effect | - |---|---| - | `"all"` (default) | every unit that reported that year — unchanged behaviour | - | `"census"` | census years only; aborts if the range holds none rather than returning nothing | - | `"consistent"` | only units reporting in *every* requested year — a balanced panel | - -* **Regardless of mode**, every result now carries `provenance$coverage` with - per-year `n_units_reporting`, `n_units_expected` and `is_census_year`, plus - `provenance$coverage_mode`. `cog_explain()` prints a "Reporting coverage" - section. So the default mode can no longer mislead silently. -* `is_census_year` is a statement about the **survey calendar**, never a claim - of completeness: FY1967 is a census year in which only 97 of Wisconsin's 608 - cities report. `n_units_reporting` is the number that tells the truth. -* On `cog_peer_compare()` the target is exempt from `"consistent"` balancing — - it is the subject of the comparison, not a member of the cohort — and the - `summary_*` quantiles are computed after the filter, so they describe the - cohort actually returned. `n_units_reporting` counts peers only, against the - cohort size. -* On `cog_find_peers()`, `coverage` governs the cohort **vintage** when `year` - is `NULL`: `"census"` snaps to the most recent census year with an observed - population, so a cohort is not built from a sample year in which most of the - candidate universe is absent. - -## `complete = TRUE`: absent cells, labelled with why they are absent - -* `cog_spending()` and `cog_revenue()` gain `complete`, defaulting to `FALSE` - (today's behaviour). With `complete = TRUE` the requested grid is filled - from the corpus's `code_set` table and every row carries a new - `value_source` column: - - | `value_source` | meaning | `amt_nominal` | - |---|---|---| - | `reported` | the corpus carries this cell | as published | - | `census_zero` | dense-source year (≤ FY2011), cell absent — Census published `$0` | `0` | - | `not_reported` | sparse-source year (≥ FY2012), cell absent — unknown | `NA` | - - The `NA` is deliberate and is the whole point: filling a modern absence - with `0` would invent data, which is precisely the error the corpus's - representation contract exists to prevent. -* This restores information the reader lost when the corpus was sparsified - (`SB194`, cog_pipeline#64) — a wide-era query whose cells were all `$0` - had begun returning nothing at all — and improves on what came before it, - since the pre-sparsification corpus could not distinguish a published zero - from an unreported cell either. -* The grid is scoped to each government's **own type**, so a county is never - filled with cells only a state can report. -* Needs a corpus published from 2026-07-29 onward (when `representation` and - `code_set` began shipping); aborts with class - `uscogdata_representation_unavailable` otherwise. Gated on the manifest - listing those tables rather than on `schema_version`, which was never - bumped for the change. Not available with `recipe` or - `expenditure_concept = "total"` — neither draws its cells from `code_set`. -* `provenance$completion` reports `applied`, `rows_filled`, and the per-year - `absence_means` rule; `cog_explain()` prints a "Completion" section. - -## Corpus-wide series breaks now reach users (`corpus_break_refs`) - -* Four catalogued series breaks carry `fin_code = "ALL"` — caveats about the - corpus as a whole rather than about one item code. `series_break_refs` is - built by matching `fin_code` against the item codes in the result, and no - row's `item_code` is ever the literal `"ALL"`, so **none of them could ever - be surfaced**: `SB085` (dollar precision across the 1976/1977 boundary), - `SB087` (imputation exclusion from FY2002), `SB194` (the dense → sparse - representation change at FY2012) and `SB086` (the government id scheme - change at FY2017). -* Provenance gains `corpus_break_refs`, selected on the break-year window - alone and disjoint from `series_break_refs` by construction, so a consumer - can tell a whole-result caveat from a break in one series. `cog_explain()` - prints them under their own "Corpus-wide caveats" heading. cog-api passes - provenance through verbatim, so the field appears there without an API - change. -* `SB194` is the one that made this urgent: a query spanning FY2011 → FY2012 - crosses the boundary where an absent cell stops meaning "Census published - `$0`" and starts meaning "not reported", and until now nothing said so. - -## Bundled fixture regenerated against the sparsified corpus - -* `inst/extdata/fixture_corpus/` now tracks the corpus published on - 2026-07-29 (`pipeline_commit 83f9715`, schema v6). The wide era no longer - stores explicit zeros: FY2011 fell from 2,864,212 rows to 496,004, of - which none are `$0`. **Absence now means two different things** — in a - `dense_source` year (≤ FY2011) an absent cell means Census published `$0`; - in a `sparse_source` year (≥ FY2012) it means not reported. The corpus - carries that rule in two new tables the fixture now ships, - `representation.parquet` and `code_set.parquet`, alongside - `census_collection_coverage.parquet` and `lineage_events.parquet` - (all ten publish-tree metadata tables, up from six). Catalogued upstream - as series break `SB194`. -* `cog_categories()` gains an `assistance` spending subtype: the J-prefix - aid/benefit codes (`J19`, `J67`, `J68`, `J85`) are categorised now that - the upstream crosswalk covers every flow code carrying dollars. -* Two consequences worth knowing about, both visible in provenance rather - than in returned dollars. The harmonization block's `na_rows_excluded` - counts only rows that exist, so wide-era codes that were zero-padded no - longer appear there. Coverage-gap `suggestions` are presence-based for the - same reason, so a recipe whose component codes were all `$0` for a given - government-year is no longer suggested for it. -* `tests/testthat/test-fixture-vintage.R` pins these structural facts, so a - fixture left behind by a future publish fails loudly instead of letting the - suite pass against a corpus that no longer exists. - -## Breaking: corpus schema_version 4 (Phase P canonical ids) - -* The package now requires corpus `schema_version = 4` (`MinCorpusSchema` / - `MaxCorpusSchema` in `DESCRIPTION` are both `4`); older corpora built - against schema 3 are rejected by `cog_open()` with a clear version-mismatch - error. `canonical_govid` is now uniformly 12 characters across every - vintage the corpus covers (previously a mix of 9-char legacy ids and - 12-char FIPS ids depending on source year) — **every hardcoded - `canonical_govid` literal from a pre-Phase-P corpus is now invalid** and - must be re-resolved via `cog_gov_search()` or the new `canonical_alias` - lookup table. `canonical_fips_xwalk` gains four columns - (`legacy_govs_id`, `census_geoid`, `id_source`; `confidence` is renamed to - `pop_confidence`) and a companion `canonical_alias` table ships in the - corpus for mapping legacy/alternate ids onto the current canonical - namespace. The bundled fixture corpus (`inst/extdata/fixture_corpus/`) has - been regenerated against the Phase P publish tree, now ships the full - `canonical_fips_xwalk` and `canonical_alias` master tables alongside the - 2019-2020 long partitions, and is reproducible via - `data-raw/regenerate_fixture_corpus.R`. - -## Clearer errors when `USCOGDATA_URL` is unconfigured or returns non-JSON - -* `cog_open()` now aborts with the `uscogdata_url_not_configured` error - class when the resolved corpus URL still contains the placeholder - `REPLACE_WITH_SHARE_TOKEN` sentinel (or is empty). The message lists both - remediation paths (`Sys.setenv(USCOGDATA_URL = ...)` and - `options(uscogdata.url = ...)`) and points at the bundled fixture for - offline testing. Previously the package proceeded to fetch the placeholder - URL, cached the resulting HTML welcome page, and failed downstream with a - cryptic `jsonlite` lexical-error. -* `.fetch_or_cache_manifest()` now parses the HTTP response body before - persisting it. Non-JSON responses (login pages, 404 HTML) raise - `uscogdata_invalid_manifest` with the URL, Content-Type, and underlying - parse error — and never write to the on-disk cache. -* Manifest cache writes are now atomic (write to `manifest.json.tmp.` - in `cache_dir`, then `file.rename` over the target), so an interrupted - fetch cannot replace a previously-good cache. -* Existing caches with non-JSON content (poisoned by the prior code path) - are silently refetched instead of returning a parse error to the caller. -* Local `USCOGDATA_URL` paths whose `manifest.json` is not valid JSON now - surface the same `uscogdata_invalid_manifest` class with file context. - -## Per-capita denominators now use per-year Census F-33 population - -* `cog_spending()` and `cog_revenue()` previously divided all years' amounts - by a single ACS 2018-2022 estimate (`canonical_fips_xwalk.population_acs`), - producing biased per-capita values for time-series analysis. They now - divide by the F-33 `population` recorded on each gov-year via the new - `gov_population_yearly` view. Result tibbles gain a `pop_source` column - with values `"census_f33"` or `"unavailable"`. `notes` is updated to - concatenate multiple notes with `"; "`. - -## Peer cohorts can be set to a chosen year - -* `cog_find_peers()` adds a `year` argument (default: most recent year for - which the target has an observed population in `gov_population_yearly`). - The returned column previously named `population_acs` is now `population` - and reflects the cohort year's vintage. The cohort year is attached to the - returned tibble as `attr(x, "cohort_year")`. -* `cog_peer_compare()` now stamps a `cohort_year` column on its result (read - from the peers tibble's attribute) and records `cohort_year` plus - `cohort_govids` in provenance. When the caller supplies a bare character - vector instead of a `cog_find_peers()` result, `cohort_year` is `NA`. - -## Rollups exclude govs missing population - -* `cog_geographic_rollup(per_capita = TRUE)` drops rows whose government has - `pop_source == "unavailable"` and records the dropped govids in - `provenance$rollup$excluded_govids`. This excludes special districts - (type 4) and school districts (type 5) from per-capita rollups by design. - -## New: vignette and provenance metadata - -* New vignette `population-denominators` covers the four population sources, - the type-4/5 coverage gap, the popyear quirk, and how to build moving-window - peer cohorts manually. -* Provenance gains `transformations$per_capita$popyear_range` and - `pop_source_counts`. `cog_explain()` renders both. - -## New features - -* `cog_gov_search()` gains a **basket mode**: passing vector `name` - / `state` / `type` arguments resolves multiple place names in one - call and returns a tibble of canonical rows in input order, ready - to pipe into `cog_spending()` / `cog_revenue()`. Per-row resolution - follows an exact-then-substring matching algorithm with deterministic - disambiguation; ambiguous and missing entries are surfaced via a - sidecar audit tibble plus a single console summary message. -* New exports `cog_basket_resolution()` and `cog_basket_unresolved()` - expose the basket sidecar for iterative query refinement. - -## Breaking changes - -* The first formal of `cog_gov_search()` was renamed from `pattern` - to `name`. All existing call sites in `cog_explorer/` and the - package itself use positional first-arg, so this rename is - non-breaking in practice. Callers that pass `pattern = ...` by name - must update to `name = ...`. diff --git a/R/config.R b/R/config.R index a5534b1..37354a3 100644 --- a/R/config.R +++ b/R/config.R @@ -5,7 +5,18 @@ .uscogdata_env <- new.env(parent = emptyenv()) .uscogdata_defaults <- list( - url = "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/", + # Public HuggingFace mirror of the published corpus: CC-BY-4.0, no + # credential, CDN-backed. This is the default so `library(uscogdata)` + # followed by a verb works with zero configuration -- previously the + # default was a REPLACE_WITH_SHARE_TOKEN sentinel and no document in the + # package supplied a working URL, so a new user had no path to a session. + # + # The trailing slash is required: every consumer concatenates onto this + # (see .resolve_url(), which enforces it anyway). + # + # Override with USCOGDATA_URL or options(uscogdata.url=) to read a + # Nextcloud share or a local copy made by cog_mirror(). + url = "https://huggingface.co/datasets/civilytics/us-cog-finance/resolve/main/", cache_dir = NULL, manifest_ttl_secs = 3600L ) diff --git a/R/explain.R b/R/explain.R index 773b9a9..417d35a 100644 --- a/R/explain.R +++ b/R/explain.R @@ -11,6 +11,30 @@ #' returns `result` invisibly for chaining. `"list"` returns the raw #' provenance list (identical to `attr(result, "provenance")`). #' @return Either `result` (invisibly) or the provenance list. +#' @section Two kinds of series break: +#' Catalogued breaks reach you without being asked for, in two disjoint +#' fields, because a caveat about one series and a caveat about the whole +#' corpus are different claims: +#' +#' * **`series_break_refs`** — breaks matched against the item codes actually +#' present in this result. A break in one code you queried. +#' * **`corpus_break_refs`** — breaks catalogued with `fin_code = "ALL"`, +#' which are statements about the corpus rather than about any one code: +#' dollar precision across the 1976/1977 boundary (`SB085`), imputation +#' exclusion from FY2002 (`SB087`), the FY2012 dense-to-sparse +#' representation change (`SB194`), and the FY2017 government-identifier +#' change (`SB086`). These are selected on the break-year window alone. +#' +#' `SB194` is the one most likely to matter: a query spanning FY2011 to FY2012 +#' crosses the boundary where an absent cell stops meaning "Census published +#' $0" and starts meaning "not reported". +#' @section Other provenance blocks: +#' `transformations$units_conversion` records the `$1,000s`-to-dollars +#' multiply that every amount column has already had applied. +#' `transformations$per_capita` records the population denominator and its +#' year range. `coverage` and `coverage_mode` appear on multi-government +#' results (see [cog_geographic_rollup()]). `completion` appears when +#' `complete = TRUE`. `balance_caveats` appears on [cog_balances()] results. #' @export cog_explain <- function(result, format = c("print", "list")) { format <- match.arg(format) diff --git a/R/manifest.R b/R/manifest.R index 1161e70..07e8b67 100644 --- a/R/manifest.R +++ b/R/manifest.R @@ -28,7 +28,7 @@ "*" = "{.code Sys.setenv(USCOGDATA_URL = \"/\")}", "*" = "{.code options(uscogdata.url = \"/\")}", i = "For an offline smoke test, use the bundled fixture: {.code system.file(\"extdata/fixture_corpus\", package = \"uscogdata\")}.", - i = "For the live Civilytics corpus, request the Nextcloud share URL from the package maintainer." + i = "The public corpus is the default: unset USCOGDATA_URL to use it, or point it at a local copy made by {.code cog_mirror()}." ), class = "uscogdata_url_not_configured") } invisible(url) diff --git a/R/views.R b/R/views.R index 28bf42c..5d1c4dc 100644 --- a/R/views.R +++ b/R/views.R @@ -78,6 +78,55 @@ file %in% basename(paths) } +#' Build the SQL path expression for the partitioned `long` table. +#' +#' DuckDB cannot expand a glob over generic HTTP: there is no directory +#' listing to expand against, and `allow_asterisks_in_http_paths` only +#' forwards the literal `**/*` as a filename, which 404s. Measured against +#' the published corpus on 2026-08-08, an explicit file list returns the +#' same 46,148,034 rows the (working) `hf://` glob does, and +#' `hive_partitioning = true` still recovers `year` from the paths. +#' +#' The manifest already enumerates every partition, so we build the list +#' from it. This is host-agnostic -- Nextcloud, HuggingFace and a local +#' fixture take the same path -- where an `hf://` URL would tie the reader +#' to one vendor's protocol and still need special-casing, since manifest +#' fetching goes through httr2, which cannot speak `hf://`. +#' +#' Falls back to the glob when the manifest carries no partition list: a +#' hand-built manifest in a test (see test-views.R) or a corpus predating +#' the field. Both are local, where globbing works. +#' @noRd +.long_files_sql <- function(url, manifest) { + parts <- manifest$files$long_partitions %||% list() + if (length(parts) == 0L) { + return(.sql_lit_chr(paste0(url, "data/long/**/*.parquet"))) + } + paths <- vapply(parts, function(p) as.character(p$path), character(1)) + paste0("[", .sql_lit_chr(paste0(url, paths)), "]") +} + +#' Substitute the corpus-location tokens in a view's SQL text. +#' +#' One place knows the token vocabulary. `.register_views()` and the tests +#' that execute a view file directly both route through here. This exists +#' because four test sites had hand-rolled the `{url}` substitution -- one +#' of them commented as doing it "exactly as .register_views() does" -- and +#' every one of them broke the moment a second token was introduced. +#' +#' `{long_files}` must be substituted BEFORE `{url}`: it expands to a string +#' that itself contains the url, so the reverse order leaves the token in +#' place and DuckDB's parser fails on the brace. +#' +#' `manifest` defaults to empty, which routes `.long_files_sql()` to its glob +#' fallback -- correct for the local temp corpora the direct-execution tests +#' build. +#' @noRd +.render_view_sql <- function(sql, url, manifest = list()) { + sql <- gsub("\\{long_files\\}", .long_files_sql(url, manifest), sql, fixed = FALSE) + gsub("\\{url\\}", url, sql, fixed = FALSE) +} + #' Register DuckDB views from inst/sql/ SQL files #' @noRd .register_views <- function(con, url, manifest) { @@ -91,7 +140,7 @@ !.corpus_has_table(manifest, .representation_view_files[[base]])) next if (base %in% .balance_view_files && !.corpus_has_balance_subtype(con)) next sql <- paste(readLines(f, warn = FALSE), collapse = "\n") - sql <- gsub("\\{url\\}", url, sql, fixed = FALSE) + sql <- .render_view_sql(sql, url, manifest) DBI::dbExecute(con, sql) } } diff --git a/README.md b/README.md index 16cbdd5..7590d76 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,116 @@ # uscogdata -Curated R reader for the Civilytics US Census of Governments finance corpus. + +[![r-universe](https://civilytics.r-universe.dev/badges/uscogdata)](https://civilytics.r-universe.dev/uscogdata) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md) + -Provides unit-level financial profiles, geographic rollups, and peer comparisons -with auditable provenance and built-in cross-vintage correctness. Reads the -published corpus (Hive-partitioned parquet + manifest.json) directly from -Nextcloud via DuckDB httpfs — no local bulk downloads required. +A curated R reader for the Civilytics US Census of Governments finance corpus — +every dollar that US state, county, municipal and township governments reported +raising and spending, from **FY1967 to FY2024**, in one queryable place. -## Status +The Census of Governments is the only nationwide source for local government +finance, and it is hard to use: item codes change meaning across vintages, +government identifiers were renumbered in 2017, and an absent value means +"published zero" in one era and "not reported" in the next. This package +handles each of those problems, and it tells you when it has — every result +carries provenance describing what was converted, what was aggregated, and +which known series breaks intersect your query. -Under active development (Phase 2 of the cog_pipeline project). See -`../cog_pipeline/docs/reader-specification.md` for the reader contract this -package implements. +**Scope:** government types 0–3 (state, county, municipality, township). +56 fiscal years, 46,148,034 rows, 190.6 MB. There is no source data for FY1968 +or FY1969. Special districts (type 4) and school districts (type 5) are +excluded pending validation. -## Installation +## Where the data comes from + +The corpus is published and documented at the **[US Census of Governments +Finance API](https://pages.civilytics.org/cog-api/)**. Start there for how the +data was built, how the identifier and item-code reconciliation works, and what +the corpus does and does not cover. + +- **[API documentation and walkthroughs](https://pages.civilytics.org/cog-api/)** + — reference, data dictionary, and worked examples such as the + [Southern states guide](https://pages.civilytics.org/cog-api/cog-api-south-guide.html) +- **[Live API](https://cog-api.civilytics.org/api/v1/)** — the same corpus over + HTTP, for Tableau, Python, or anything that isn't R +- **[Bulk corpus on Hugging Face](https://huggingface.co/datasets/civilytics/us-cog-finance)** + — CC-BY-4.0; the same parquet files this package reads +- **[Census Bureau source data](https://www.census.gov/programs-surveys/gov-finances.html)** + — the underlying public files + +## Install ```r -# pak::pkg_install("gitea.civilytics.org/Civilytics/uscogdata") +install.packages("uscogdata", + repos = c("https://civilytics.r-universe.dev", + "https://cloud.r-project.org")) ``` +Or from source: + +```r +pak::pkg_install("git::https://gitea.civilytics.org/Civilytics/uscogdata.git") +``` + +## Quickstart + +No configuration, no credentials, no download. The package reads the published +corpus over HTTPS by default. + +```r +library(uscogdata) + +# Resolve a place name to a canonical government id +madison <- cog_gov_search(name = "Madison", state = "WI", type = 2) +madison$canonical_govid +#> [1] "552025209777" + +# Police spending, inflation-adjusted and per capita +spend <- cog_spending( + madison$canonical_govid, + years = 2012:2022, + category = "Police", + per_capita = TRUE, + adjust_to_year = 2023 +) + +# What did that result do to the numbers, and what should you know about them? +cog_explain(spend) +``` + +`years` is required — there is no implicit full-history default. + +## Two ways to read the corpus + +| | Remote (default) | Mirrored | +|---|---|---| +| Setup | none | `cog_mirror(dest)`, 190.6 MB once | +| Disk used | **0 MB** — HTTP range requests only | 190.6 MB | +| Per query | ~4 s (one government, one year)
~6 s (one government, 23 years) | local speed | +| Good for | trying it out, teaching, one-off questions | repeated analysis, offline work, reproducibility | + +Nothing is written to disk in remote mode: DuckDB fetches the parquet footer, +works out which row groups it needs, and reads only those. Nothing is cached +between sessions either, so every query goes back to the network. + +The default points at a public HuggingFace mirror of the corpus. If you would +rather not depend on a third party — for reproducibility, for an air-gapped +environment, or on principle — **the escape hatch is one function call**: + +```r +cog_mirror("~/cog-corpus") +Sys.setenv(USCOGDATA_URL = "~/cog-corpus/") +``` + +After that, nothing in your analysis touches an external service. + +### Configuration + +- `USCOGDATA_URL` — corpus root: an HTTPS URL or a local path, **trailing slash required** +- `USCOGDATA_CACHE_DIR` — where the manifest is cached (default: user cache dir) +- `USCOGDATA_MANIFEST_TTL_SECS` — manifest re-fetch interval (default 3600) + ## Amounts are in full US dollars Every amount column this package returns — `amt_nominal`, `amt_real`, @@ -29,111 +121,127 @@ own `amt` column preserves that. The verbs multiply by 1000 on the way out, so you never have to. The conversion is recorded in every result: ```r -r <- cog_spending("552025209777", 2020L) -attr(r, "provenance")$transformations$units_conversion -#> $applied TRUE $source_unit "$1,000s (raw Census)" $target_unit "$USD" $multiplier 1000 +attr(spend, "provenance")$transformations$units_conversion +#> $applied TRUE +#> $source_unit "$1,000s (raw Census)" +#> $target_unit "$USD" +#> $multiplier 1000 ``` **Do not multiply again.** If you have read elsewhere that COG amounts are in -`$1,000s` — true of the raw corpus, and of `cog_explorer`'s conventions doc — -that rule does not apply to anything a `cog_*()` verb hands you. Applying it -twice overstates every figure by 1000x, and the result looks plausible rather -than obviously wrong. +`$1,000s` — which is true of the raw Census files and of the corpus's own `amt` +column — that rule does not apply to anything a `cog_*()` verb hands you. +Applying it twice overstates every figure by 1000x, and the result looks +plausible rather than obviously wrong. -## Configuration +## Concepts worth understanding before you publish a number -- `USCOGDATA_URL` — corpus root URL (public Nextcloud share, trailing slash) -- `USCOGDATA_CACHE_DIR` — optional override for the manifest cache directory -- `USCOGDATA_MANIFEST_TTL_SECS` — optional manifest re-fetch TTL (default 3600) - -## Primary vs Direct vs Total spending +### Primary vs Direct vs Total spending `cog_spending(..., expenditure_concept = c("primary", "direct", "total"))` -controls whose spending a result counts. Concepts are defined as sets of the -crosswalk's `spend_subtype` values — never item-code first letters, which -cannot classify correctly (the letter `Y` alone spans revenue, expenditure, -and balance codes): +controls *whose* spending a result counts. Concepts are defined as sets of the +crosswalk's `spend_subtype` values, never item-code first letters — the letter +`Y` alone spans revenue, expenditure and balance codes. -- `"primary"` (the default) is the government's own service provision: - current operations, capital outlay, and assistance payments. -- `"direct"` is Census's published Direct Expenditure: `primary` plus - interest on debt and insurance trust benefit payments (e.g. pensions). -- `"total"` additionally adds the intergovernmental leg — money handed to - other governments to spend (`M`/`L` codes plus `Q11`/`Q12`/`Q18` state - payments to school systems) — which is meaningful for describing one - government's own budget over time, but double-counts when summed across - governments (a state's payment to a county is the same dollar the county - reports as its own direct spending). +- **`"primary"`** (default) — the government's own service provision: current + operations, capital outlay, assistance payments. +- **`"direct"`** — Census's published Direct Expenditure: `primary` plus + interest on debt and insurance trust benefits (e.g. pensions). +- **`"total"`** — adds the intergovernmental leg, money handed to other + governments to spend. Meaningful for one government's own budget over time, + but it double-counts when summed across governments: a state's payment to a + county is the same dollar the county reports as its own direct spending. -**Rule of thumb: any figure that spans more than one government uses -`primary` or `direct`.** `cog_geographic_rollup()` and `cog_peer_compare()` -enforce this by refusing `expenditure_concept = "total"`. See -`vignette("total-spending", package = "uscogdata")` for the full -explanation with worked examples. +**Rule of thumb: any figure spanning more than one government uses `primary` +or `direct`.** `cog_geographic_rollup()` and `cog_peer_compare()` enforce that +by refusing `"total"` outright. Worked examples in +`vignette("total-spending", package = "uscogdata")`. -## General vs Total revenue +### General vs Total revenue -`cog_revenue(..., revenue_concept = c("general", "total"))` selects between -Census's two published revenue concepts, again defined as crosswalk -`revenue_subtype` sets rather than item-code prefixes: +`cog_revenue(..., revenue_concept = c("general", "total"))`: -- `"general"` (the default) is Census **General Revenue**: own-source - (taxes, charges, miscellaneous) plus federal, state and local - intergovernmental aid. -- `"total"` is Census **Total Revenue**: `general` plus utility revenue - (`A91`–`A94`), liquor store revenue (`A90`), and insurance trust revenue - (unemployment and workers' compensation `Y` codes plus the - employee-retirement `X` codes). +- **`"general"`** (default) — Census General Revenue: own-source taxes, + charges and miscellaneous, plus federal, state and local aid. +- **`"total"`** — General plus utility revenue (`A91`–`A94`), liquor store + revenue (`A90`), and insurance trust revenue. -The manual defines the first by subtracting the other three from the second, -so the two are related by Census's own identity: +Census defines these by its own identity: ``` Total Revenue = General + Utility + Liquor Store + Insurance Trust ``` -Two things worth knowing before switching to `"total"`: +Two things to know before switching to `"total"`. **Utility revenue is large +for cities** — measured on the bundled fixture, utility plus liquor store is +15.9% of city revenue, against 1.2% for states and 1.7% for counties. And the +**employee-retirement (`X`) codes stop at FY2016**, when those systems moved to +the separate Annual Survey of Public Pensions, so a `"total"` series steps down +at the FY2016/FY2017 boundary for reasons of collection scope, not revenue +(series breaks `SB197`–`SB209`). -- **Utility revenue is large for cities.** Measured on the bundled fixture, - utility plus liquor store revenue is 15.9% of city (type 2) revenue, versus - 1.2% for states and 1.7% for counties. `general` excludes it by definition. -- **The employee-retirement (`X`) codes stop at FY2016**, when those systems - moved out of the annual finance file into the separate Annual Survey of - Public Pensions. A `"total"` series therefore steps down at the - FY2016/FY2017 seam for reasons of collection scope, not revenue (series - breaks `SB197`–`SB202`, in the corpus's `series_breaks` table). +### Reporting coverage: the Census is only sometimes a census -## Developer notes +**The Census of Governments is a complete enumeration only in years ending in +2 and 7.** Every other year is a sample, and the sample varies enormously — +measured on the bundled fixture, Wisconsin's 608-city universe rolls up 597 +governments in FY2012 and 112 in FY2019. -### Testing - -The package ships a bundled fixture corpus at `inst/extdata/fixture_corpus/` — -a 15 MB four-year slice (2011, 2012, 2019, 2020) of the full corpus covering -all 50 states. `tests/testthat/setup.R` automatically points `USCOGDATA_URL` -at this fixture, so the full test suite runs offline with no network -dependency: +A statewide total resting on a fifth of the universe looks exactly like one +resting on all of it, so every multi-government result now says which it is: ```r -devtools::test() # uses bundled fixture, no credentials required +attr(rollup, "provenance")$coverage # per-year n_units_reporting, is_census_year ``` -### Releasing against the live corpus +`cog_geographic_rollup()`, `cog_peer_compare()` and `cog_find_peers()` take a +`coverage` argument — `"all"` (default), `"census"` (census years only), or +`"consistent"` (only units reporting in every requested year, a balanced +panel). -Before cutting a release, run the test suite against the published corpus to -catch any drift between the fixture and the real data: +`n_units_reporting` is **category-conditional**, and it is not a response rate. A government that was surveyed and genuinely spends +nothing in the requested category is indistinguishable from one never surveyed. + +### Absent cells mean two different things + +Before FY2012, an absent cell means Census published `$0`. From FY2012 on, it +means not reported. `cog_spending(..., complete = TRUE)` fills the requested +grid and labels every row with which it is, via `value_source`: + +| `value_source` | meaning | `amt_nominal` | +|---|---|---| +| `reported` | the corpus carries this cell | as published | +| `census_zero` | dense-source year (≤ FY2011), absent — Census published `$0` | `0` | +| `not_reported` | sparse-source year (≥ FY2012), absent — unknown | `NA` | + +That `NA` is deliberate. Filling a modern absence with `0` would invent data. + +### Series breaks surface on their own + +Catalogued breaks that intersect your query appear in provenance whether or not +you went looking for them — `series_break_refs` for breaks in a specific item code, and +`corpus_break_refs` for caveats about the corpus as a whole (dollar precision +across the 1976/1977 boundary, the FY2017 identifier change, the FY2012 +dense→sparse representation change). `cog_explain()` prints both. + +## How to cite ```r -Sys.setenv(USCOGDATA_URL = "") -devtools::test() +citation("uscogdata") ``` -When the live-corpus run is clean, strip the fixture from the built package by -adding this line to `.Rbuildignore`: +The corpus itself is published under CC-BY-4.0. Cite it as: -``` -^inst/extdata/fixture_corpus$ -``` +> Civilytics Consulting. US Census of Governments finance corpus. +> https://huggingface.co/datasets/civilytics/us-cog-finance -The test suite is URL-agnostic — `setup.R` falls back to `USCOGDATA_URL` when -the bundled fixture is absent, so no test code changes are needed for the -release run or after stripping the fixture. +## Contributing + +Development happens on [Gitea](https://gitea.civilytics.org/Civilytics/uscogdata); +[GitHub](https://github.com/civilytics/uscogdata) is a mirror that accepts +issues and pull requests. See [CONTRIBUTING.md](CONTRIBUTING.md) for how a +patch gets from there to here. + +## License + +MIT © Civilytics Consulting LLC. See [LICENSE.md](LICENSE.md). diff --git a/_pkgdown.yml b/_pkgdown.yml index 1541211..057bc65 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,4 +1,5 @@ -url: ~ +url: https://civilytics.r-universe.dev/uscogdata + template: bootstrap: 5 @@ -15,11 +16,26 @@ reference: - cog_gov_search - cog_basket_resolution - cog_basket_unresolved - - title: Session + - title: Comparison & aggregation + desc: Peer cohorts and geographic aggregates. contents: - - has_keyword("internal") + - cog_find_peers + - cog_peer_compare + - cog_geographic_rollup + - title: Corpus metadata + desc: > + What the corpus contains, where a given result came from, and how to + hold a local copy of it. + contents: + - cog_categories + - cog_recipes + - cog_manifest + - cog_explain + - cog_mirror articles: - - title: Getting started + - title: Concepts navbar: ~ - contents: [] + contents: + - total-spending + - population-denominators diff --git a/inst/sql/10-long.sql b/inst/sql/10-long.sql index 5cf5fcc..d33f0c4 100644 --- a/inst/sql/10-long.sql +++ b/inst/sql/10-long.sql @@ -1,3 +1,7 @@ CREATE OR REPLACE VIEW long AS SELECT * -FROM read_parquet('{url}data/long/**/*.parquet', hive_partitioning = true); +-- {long_files} carries its own quoting: a bracketed list of every partition +-- the manifest enumerates, or a single quoted glob on fallback. Do NOT wrap +-- it in quotes. See .long_files_sql() in R/views.R for why a glob alone +-- cannot work over HTTP. +FROM read_parquet({long_files}, hive_partitioning = true); diff --git a/man/cog_explain.Rd b/man/cog_explain.Rd index d67eb71..6a4e712 100644 --- a/man/cog_explain.Rd +++ b/man/cog_explain.Rd @@ -21,3 +21,33 @@ Prints the structured provenance attached to a tibble returned by any `cog_*` verb, or returns it as a list for downstream use (MCP tools, dashboards, JSON export). } +\section{Two kinds of series break}{ + +Catalogued breaks reach you without being asked for, in two disjoint +fields, because a caveat about one series and a caveat about the whole +corpus are different claims: + +* **`series_break_refs`** — breaks matched against the item codes actually + present in this result. A break in one code you queried. +* **`corpus_break_refs`** — breaks catalogued with `fin_code = "ALL"`, + which are statements about the corpus rather than about any one code: + dollar precision across the 1976/1977 boundary (`SB085`), imputation + exclusion from FY2002 (`SB087`), the FY2012 dense-to-sparse + representation change (`SB194`), and the FY2017 government-identifier + change (`SB086`). These are selected on the break-year window alone. + +`SB194` is the one most likely to matter: a query spanning FY2011 to FY2012 +crosses the boundary where an absent cell stops meaning "Census published +$0" and starts meaning "not reported". +} + +\section{Other provenance blocks}{ + +`transformations$units_conversion` records the `$1,000s`-to-dollars +multiply that every amount column has already had applied. +`transformations$per_capita` records the population denominator and its +year range. `coverage` and `coverage_mode` appear on multi-government +results (see [cog_geographic_rollup()]). `completion` appears when +`complete = TRUE`. `balance_caveats` appears on [cog_balances()] results. +} + diff --git a/plans/2026-08-08-public-release.md b/plans/2026-08-08-public-release.md new file mode 100644 index 0000000..9134625 --- /dev/null +++ b/plans/2026-08-08-public-release.md @@ -0,0 +1,1091 @@ +# uscogdata 0.3.0 Public Release Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `uscogdata` installable and usable by a stranger — fix the corpus-unreachable defect, correct release metadata, and rewrite README and NEWS for a first public release. + +**Architecture:** Two code changes fix the P0 (manifest-driven file enumeration replacing an HTTP-incompatible glob; a working public default URL). Everything else is metadata, packaging config, and documentation. Distribution mechanics (Gitea public, GitHub mirror, r-universe) are **out of scope for this plan** — they follow after checks are green. + +**Tech Stack:** R (>= 4.1), DuckDB 1.5.5 via `duckdb`/`DBI`, `httr2`, `jsonlite`, `cli`, testthat 3e, pkgdown, roxygen2 7.3.3. + +**Design spec:** `specs/2026-08-08-public-release-design.md` + +## Global Constraints + +- Package license is **MIT**. Copyright holder is **Civilytics Consulting LLC**. +- Author of record: **Jared E. Knowles**, `jared@civilytics.com`, ORCID **0000-0003-0005-9478**, roles `aut`/`cre`. Civilytics Consulting LLC is `cph`/`fnd`. +- Public corpus base URL (note the trailing slash, which `.resolve_url()` requires): + `https://huggingface.co/datasets/civilytics/us-cog-finance/resolve/main/` +- Published corpus is **schema_version 7**; bundled fixture is **schema_version 6**. `.validate_schema()` accepts `c(4L, 5L, 6L, 7L)` and that list does not change in this plan. +- Corpus facts for documentation, measured 2026-08-08: **46,148,034 rows**, **56 partitions**, **190.6 MB**, government types **0–3**, **FY1967–FY2024** (no source data FY1968, FY1969). +- The bundled fixture at `inst/extdata/fixture_corpus/` **ships in the release**. Never add it to `.Rbuildignore`. +- Every amount column a verb returns is in **full US dollars**, already multiplied by 1000. Documentation must never tell a user to apply the $1,000s rule to verb output. +- Run the full suite with `devtools::test()` from the package root. It uses the bundled fixture and requires no network or credentials. + +--- + +## File Structure + +**Modified — code** +- `R/views.R` — gains `.long_files_sql()`; `.register_views()` substitutes a second token. This is the only file that knows how the `long` table's paths are built. +- `inst/sql/10-long.sql` — the one file containing a glob. Becomes token-driven. +- `R/config.R` — default corpus URL. +- `R/manifest.R` — `.check_url_configured()` guidance text only; the sentinel check itself is unchanged. + +**Modified — metadata/packaging** +- `DESCRIPTION`, `LICENSE`, `.Rbuildignore`, `_pkgdown.yml` + +**Created** +- `LICENSE.md`, `CONTRIBUTING.md` +- `tests/testthat/test-long-files.R` — unit tests for enumeration +- `tests/testthat/test-live-corpus.R` — network-gated integration test + +**Rewritten** +- `README.md`, `NEWS.md` + +**Deliberately untouched:** every verb file (`R/spending.R`, `R/revenue.R`, `R/balances.R`, `R/rollup.R`, `R/peers.R`, `R/search.R`), `R/mirror.R`, and all other `inst/sql/*.sql`. The enumeration fix is confined to view registration by design. + +--- + +## Task 1: Manifest-driven partition enumeration + +The P0 defect, part one. `inst/sql/10-long.sql` globs `{url}data/long/**/*.parquet`. DuckDB 1.5.5 refuses globs over generic HTTP, and `allow_asterisks_in_http_paths` does not help — it forwards the literal `**/*` as a filename and 404s, because HTTP exposes no directory listing. The manifest already enumerates every partition under `files$long_partitions[]`. + +**Files:** +- Modify: `R/views.R` (add helper; `.register_views()` at the `gsub` line) +- Modify: `inst/sql/10-long.sql:3` +- Test: `tests/testthat/test-long-files.R` (create) + +**Interfaces:** +- Consumes: `.sql_lit_chr(x)` from `R/spending.R:553` — quotes each element, escapes `'` by doubling, joins with `,` and **no space**. `%||%` from `R/manifest.R:158`. +- Produces: `.long_files_sql(url, manifest)` returning a single SQL string — either a bracketed list literal `['a','b']` or, on fallback, a single quoted glob `'…/**/*.parquet'`. Task 3 relies on this being the only place partition paths are constructed. + +**Critical constraint:** `tests/testthat/test-views.R:322` calls `.register_views(con, url, manifest = list(schema_version = 4L))` — a manifest with **no `files` element at all**. The helper must not error on it. The glob fallback exists for exactly this case and for local paths, where globbing works fine. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/testthat/test-long-files.R`: + +```r +test_that(".long_files_sql enumerates every partition the manifest lists", { + manifest <- list(files = list(long_partitions = list( + list(year = 2011L, path = "data/long/year=2011/part-0.parquet"), + list(year = 2012L, path = "data/long/year=2012/part-0.parquet") + ))) + expect_equal( + uscogdata:::.long_files_sql("https://example.org/corpus/", manifest), + paste0( + "['https://example.org/corpus/data/long/year=2011/part-0.parquet',", + "'https://example.org/corpus/data/long/year=2012/part-0.parquet']" + ) + ) +}) + +test_that(".long_files_sql falls back to the glob when no partition list is present", { + # test-views.R registers views with a hand-built manifest that has no + # `files` element. That must keep working: the glob is valid for the + # local paths such a manifest is used with. + expect_equal( + uscogdata:::.long_files_sql("/tmp/corpus/", list(schema_version = 4L)), + "'/tmp/corpus/data/long/**/*.parquet'" + ) + expect_equal( + uscogdata:::.long_files_sql("/tmp/corpus/", list(files = list(long_partitions = list()))), + "'/tmp/corpus/data/long/**/*.parquet'" + ) +}) + +test_that("the enumerated list matches the bundled fixture's partition count", { + skip_if_no_corpus() + m <- jsonlite::fromJSON( + file.path(fixture_corpus_path(), "manifest.json"), simplifyVector = FALSE + ) + out <- uscogdata:::.long_files_sql(fixture_corpus_path(), m) + expect_equal( + lengths(regmatches(out, gregexpr("part-0\\.parquet", out))), + length(m$files$long_partitions) + ) +}) + +test_that("registered `long` view reads through the enumerated list", { + skip_if_no_corpus() + with_fixture_corpus({ + con <- uscogdata:::.ensure_session() + n <- DBI::dbGetQuery(con, "SELECT count(*) AS n FROM long")$n + expect_gt(n, 0) + yrs <- DBI::dbGetQuery(con, "SELECT DISTINCT year FROM long ORDER BY year")$year + expect_true(all(c(2011, 2012, 2019, 2020) %in% yrs)) + }) +}) +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `Rscript -e 'devtools::test(filter = "long-files")'` +Expected: FAIL — `could not find function ".long_files_sql"` on the first three; the fourth may pass already (it exercises the glob against a local fixture, which works). + +- [ ] **Step 3: Add the helper to `R/views.R`** + +Insert immediately above `#' Register DuckDB views from inst/sql/ SQL files`: + +```r +#' Build the SQL path expression for the partitioned `long` table. +#' +#' DuckDB cannot expand a glob over generic HTTP: there is no directory +#' listing to expand against, and `allow_asterisks_in_http_paths` only +#' forwards the literal `**/*` as a filename, which 404s. Measured against +#' the published corpus on 2026-08-08, an explicit file list returns the +#' same 46,148,034 rows the (working) `hf://` glob does, and +#' `hive_partitioning = true` still recovers `year` from the paths. +#' +#' The manifest already enumerates every partition, so we build the list +#' from it. This is host-agnostic -- Nextcloud, HuggingFace and a local +#' fixture take the same path -- where an `hf://` URL would tie the reader +#' to one vendor's protocol and still need special-casing, since manifest +#' fetching goes through httr2, which cannot speak `hf://`. +#' +#' Falls back to the glob when the manifest carries no partition list: a +#' hand-built manifest in a test (see test-views.R) or a corpus predating +#' the field. Both are local, where globbing works. +#' @noRd +.long_files_sql <- function(url, manifest) { + parts <- manifest$files$long_partitions %||% list() + if (length(parts) == 0L) { + return(.sql_lit_chr(paste0(url, "data/long/**/*.parquet"))) + } + paths <- vapply(parts, function(p) as.character(p$path), character(1)) + paste0("[", .sql_lit_chr(paste0(url, paths)), "]") +} +``` + +- [ ] **Step 4: Substitute the new token in `.register_views()`** + +In `R/views.R`, replace this line: + +```r + sql <- gsub("\\{url\\}", url, sql, fixed = FALSE) +``` + +with: + +```r + sql <- gsub("\\{long_files\\}", .long_files_sql(url, manifest), sql, fixed = FALSE) + sql <- gsub("\\{url\\}", url, sql, fixed = FALSE) +``` + +Order matters: `{long_files}` expands to a string containing the url, so it must be substituted first or the `{url}` pass would have nothing to do and the token would survive. + +- [ ] **Step 5: Change the SQL to use the token** + +`inst/sql/10-long.sql` line 3 becomes: + +```sql +FROM read_parquet({long_files}, hive_partitioning = true); +``` + +Note there are **no surrounding quotes** — `.long_files_sql()` returns its own quoting, whether a bracketed list or a single quoted glob. + +- [ ] **Step 6: Run the new tests** + +Run: `Rscript -e 'devtools::test(filter = "long-files")'` +Expected: PASS, all four. + +- [ ] **Step 7: Run the full suite for regressions** + +Run: `Rscript -e 'devtools::test()'` +Expected: PASS. Pay particular attention to `test-views.R` — it is the file that exercises `.register_views()` with a `files`-less manifest. + +- [ ] **Step 8: Commit** + +```bash +git add R/views.R inst/sql/10-long.sql tests/testthat/test-long-files.R +git commit -m "fix: enumerate long partitions from the manifest instead of globbing + +DuckDB cannot expand a glob over generic HTTP -- no directory listing -- +so every remote corpus read failed. Only local paths worked, which is how +the API and the test fixture run, so nothing caught it. + +Measured against the published corpus: the explicit list returns the same +46,148,034 rows, with hive_partitioning still recovering year." +``` + +--- + +## Task 2: Ship a working default corpus URL + +The P0 defect, part two. The default is the literal `REPLACE_WITH_SHARE_TOKEN` sentinel and no file in the repo supplies a real URL, so a new user has no path to a working session. + +**Files:** +- Modify: `R/config.R:7` +- Modify: `R/manifest.R` (the `i =` guidance line in `.check_url_configured()`) +- Modify: `tests/testthat/test-manifest.R:11`, `:38` +- Test: `tests/testthat/test-config.R` (add cases) + +**Interfaces:** +- Consumes: `.cfg("url")`, `.resolve_url()` from `R/config.R`. +- Produces: a `.uscogdata_defaults$url` that is a real, reachable, credential-free URL. Task 3's integration test depends on this being the default. + +**Critical constraint:** `tests/testthat/test-manifest.R` hardcodes the placeholder at lines 11 and 38 and asserts `cog_open()` aborts with `uscogdata_url_not_configured`. Changing the default **breaks those two tests** and they must be updated in this task. The test at line 26 passes an explicit sentinel-bearing URL via env var — it keeps passing untouched, and it is what proves the sentinel guard still works. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/testthat/test-config.R`: + +```r +test_that("the default corpus URL is real, not a placeholder", { + withr::with_envvar(c(USCOGDATA_URL = NA), { + withr::with_options(list(uscogdata.url = NULL), { + url <- uscogdata:::.resolve_url() + expect_false(grepl("REPLACE_WITH", url, fixed = TRUE)) + expect_match(url, "^https://", perl = TRUE) + expect_match(url, "/$", perl = TRUE) + }) + }) +}) + +test_that("an explicitly-set sentinel URL still aborts", { + # The guard must survive the default change: a user who half-edited a + # copied config still gets the actionable error. + withr::with_envvar( + c(USCOGDATA_URL = "https://other.example/s/REPLACE_WITH_SHARE_TOKEN/x/"), { + expect_error( + uscogdata:::.check_url_configured(uscogdata:::.resolve_url()), + class = "uscogdata_url_not_configured" + ) + }) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: FAIL on the first test — the resolved default still contains `REPLACE_WITH`. + +- [ ] **Step 3: Change the default** + +`R/config.R`, in `.uscogdata_defaults`: + +```r +.uscogdata_defaults <- list( + # Public HuggingFace mirror of the published corpus: CC-BY-4.0, no + # credential, CDN-backed. Chosen as the default so `library(uscogdata)` + # followed by a verb works with zero configuration. Trailing slash is + # required -- every consumer concatenates onto this (see .resolve_url()). + url = "https://huggingface.co/datasets/civilytics/us-cog-finance/resolve/main/", + cache_dir = NULL, + manifest_ttl_secs = 3600L +) +``` + +- [ ] **Step 4: Update the stale guidance line** + +In `R/manifest.R`, inside `.check_url_configured()`, replace: + +```r + i = "For the live Civilytics corpus, request the Nextcloud share URL from the package maintainer." +``` + +with: + +```r + i = "The public corpus is the default; unset USCOGDATA_URL to use it, or point it at a local copy from {.code cog_mirror()}." +``` + +- [ ] **Step 5: Correct the two now-inaccurate test names** + +Both affected tests in `tests/testthat/test-manifest.R` set `USCOGDATA_URL` **explicitly** via `withr::with_envvar` before calling `cog_open()`, so they keep passing unchanged. Only their wording becomes wrong — the sentinel URL is no longer the default. + +Line 7, rename the test: + +```r +test_that("cog_open aborts with actionable error when URL contains the sentinel", { +``` + +Lines 11 and 38, rename the variable and say why it is still here: + +```r + # No longer the package default (that is the public HF corpus). This is a + # user who copied a config template and did not finish editing it. + sentinel_url <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/" +``` + +Update the two `withr::with_envvar(c(USCOGDATA_URL = placeholder), ...)` call sites in those tests to use `sentinel_url`. No functional change — do not alter the assertions. + +- [ ] **Step 6: Run the affected files** + +Run: `Rscript -e 'devtools::test(filter = "config|manifest")'` +Expected: PASS. + +- [ ] **Step 7: Run the full suite** + +Run: `Rscript -e 'devtools::test()'` +Expected: PASS. `setup.R` points `USCOGDATA_URL` at the bundled fixture for the whole suite, so the default change should not affect any other file. + +- [ ] **Step 8: Commit** + +```bash +git add R/config.R R/manifest.R tests/testthat/test-config.R tests/testthat/test-manifest.R +git commit -m "feat: default to the public corpus so the package works unconfigured + +The default was a REPLACE_WITH_SHARE_TOKEN sentinel and no file in the +repo supplied a working URL, so a new user had no path to a session. The +sentinel guard stays for half-edited configs." +``` + +--- + +## Task 3: Live-corpus integration test + +Nothing in the suite exercises a remote corpus — that is why the P0 survived. This test is network-gated so it skips in offline CI but runs on demand and before release. + +**Files:** +- Test: `tests/testthat/test-live-corpus.R` (create) + +**Interfaces:** +- Consumes: `.long_files_sql()` (Task 1), the default URL (Task 2), and the public verbs `cog_gov_search()`, `cog_spending()`, `cog_explain()`. +- Produces: nothing consumed downstream. + +- [ ] **Step 1: Write the test** + +Create `tests/testthat/test-live-corpus.R`: + +```r +# Network-gated. Set USCOGDATA_LIVE_TEST=true to run. +# +# This file exists because the P0 fixed in this release -- no remote corpus +# was readable at all -- survived precisely because every other test path +# used a LOCAL corpus (the bundled fixture) and so did the API in +# production. Nothing ever exercised the code the way a new user does. +skip_live <- function() { + testthat::skip_if_not( + identical(tolower(Sys.getenv("USCOGDATA_LIVE_TEST", "")), "true"), + "live-corpus test: set USCOGDATA_LIVE_TEST=true to run" + ) +} + +test_that("the package reads the public corpus with no configuration at all", { + skip_live() + withr::with_envvar(c(USCOGDATA_URL = NA, USCOGDATA_FIXTURE_URL = NA), { + withr::with_options(list(uscogdata.url = NULL), { + uscogdata:::cog_close() + on.exit(uscogdata:::cog_close(), add = TRUE) + + g <- cog_gov_search(name = "Madison", state = "WI", type = 2) + expect_gt(nrow(g), 0) + + s <- cog_spending(g$canonical_govid[1], years = 2022) + expect_gt(nrow(s), 0) + expect_true(all(c("amt_nominal", "year", "category") %in% names(s))) + + # Amounts are full dollars, already x1000. A city's total annual + # spending is millions, not thousands -- this catches a regression + # that reintroduced the double conversion. + expect_gt(sum(s$amt_nominal, na.rm = TRUE), 1e6) + + p <- attr(s, "provenance") + expect_true(isTRUE(p$transformations$units_conversion$applied)) + expect_equal(p$transformations$units_conversion$multiplier, 1000) + }) + }) +}) + +test_that("a full-history query spans the published range", { + skip_live() + withr::with_envvar(c(USCOGDATA_URL = NA, USCOGDATA_FIXTURE_URL = NA), { + withr::with_options(list(uscogdata.url = NULL), { + uscogdata:::cog_close() + on.exit(uscogdata:::cog_close(), add = TRUE) + + g <- cog_gov_search(name = "Madison", state = "WI", type = 2) + s <- cog_spending(g$canonical_govid[1]) + # The corpus publishes FY1967-FY2024. Any single government's span is + # narrower, but a full-history query must cross more than one decade + # -- if enumeration silently returned one partition, this fails. + expect_gt(diff(range(s$year)), 10) + }) + }) +}) +``` + +- [ ] **Step 2: Run it gated off (default) — it must skip, not fail** + +Run: `Rscript -e 'devtools::test(filter = "live-corpus")'` +Expected: SKIP on both, with the message about `USCOGDATA_LIVE_TEST`. + +- [ ] **Step 3: Run it live** + +Run: `USCOGDATA_LIVE_TEST=true Rscript -e 'devtools::test(filter = "live-corpus")'` +Expected: PASS. This is the first time the package has ever read a remote corpus successfully. If it fails, Task 1 is incomplete — do not proceed. + +- [ ] **Step 4: Record the measured verb latency** + +Run and note the wall time, which the README needs (raw scans were 1.5 s / 2.8 s; real verbs do more work): + +```bash +USCOGDATA_LIVE_TEST=true Rscript -e ' + Sys.unsetenv("USCOGDATA_URL") + library(uscogdata) + g <- cog_gov_search(name = "Madison", state = "WI", type = 2) + print(system.time(cog_spending(g$canonical_govid[1], years = 2022))) + print(system.time(cog_spending(g$canonical_govid[1]))) +' +``` + +Carry these two numbers into Task 8. Do not reuse the raw-scan figures. + +- [ ] **Step 5: Commit** + +```bash +git add tests/testthat/test-live-corpus.R +git commit -m "test: exercise the public corpus end to end, unconfigured + +The remote-read defect survived because every test path used a local +corpus. This is the only test that runs the package the way a new user +does." +``` + +--- + +## Task 4: DESCRIPTION metadata + +**Files:** +- Modify: `DESCRIPTION` + +**Interfaces:** +- Produces: `URL`/`BugReports` that Task 7 (`_pkgdown.yml`) and Task 8 (README) both reference; the `Authors@R` that `citation("uscogdata")` renders. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/testthat/test-config.R`: + +```r +test_that("DESCRIPTION carries release metadata", { + skip_if_no_source_tree("DESCRIPTION") + d <- read.dcf(source_tree_path("DESCRIPTION")) + fields <- colnames(d) + + expect_true(all(c("URL", "BugReports") %in% fields)) + expect_match(d[1, "Authors@R"], "Knowles", fixed = TRUE) + expect_match(d[1, "Authors@R"], "0000-0003-0005-9478", fixed = TRUE) + expect_match(d[1, "Authors@R"], "Civilytics Consulting LLC", fixed = TRUE) + + # The gate in .validate_schema() accepts up to 7 and the published corpus + # IS 7; DESCRIPTION must not claim otherwise. + expect_equal(as.integer(d[1, "MaxCorpusSchema"]), 7L) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: FAIL — `URL`/`BugReports` absent, `MaxCorpusSchema` is 5. + +- [ ] **Step 3: Edit DESCRIPTION** + +Replace the `Authors@R` block: + +``` +Authors@R: c( + person(c("Jared", "E."), "Knowles", + email = "jared@civilytics.com", + role = c("aut", "cre"), + comment = c(ORCID = "0000-0003-0005-9478")), + person("Civilytics Consulting LLC", role = c("cph", "fnd"))) +``` + +The given-name vector `c("Jared", "E.")` with family name `"Knowles"` matches +`merTools` exactly. That structural match is what lets ORCID and r-universe +collate both packages as one person's work — `person("Jared", "E. Knowles")` +would render identically but put the middle initial in the family-name slot. + +Add after `Description:`: + +``` +URL: https://github.com/civilytics/uscogdata, https://civilytics.r-universe.dev/uscogdata +BugReports: https://github.com/civilytics/uscogdata/issues +``` + +Change: + +``` +MaxCorpusSchema: 7 +``` + +Bump the version — this release changes user-visible behaviour (remote reads +go from broken to working; the default URL from placeholder to live corpus), +which is a minor bump, not a patch: + +``` +Version: 0.3.0 +``` + +- [ ] **Step 4: Verify the person object parses** + +Run: `Rscript -e 'print(eval(parse(text = read.dcf("DESCRIPTION")[1, "Authors@R"])))'` +Expected: prints two entries — `Jared E. Knowles [aut, cre] (ORCID: ...)` and `Civilytics Consulting LLC [cph, fnd]`. A parse error here means a malformed `person()` call. + +- [ ] **Step 5: Run the tests** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add DESCRIPTION tests/testthat/test-config.R +git commit -m "chore: release metadata -- author of record, URLs, schema ceiling + +Authors@R was an org with no human, so citation() and the r-universe +maintainer page had nothing to render. MaxCorpusSchema claimed 5 while the +code accepts 7 and the published corpus is 7." +``` + +--- + +## Task 5: License files + +**Files:** +- Modify: `LICENSE` +- Create: `LICENSE.md` + +- [ ] **Step 1: Generate both files** + +Run: `Rscript -e 'usethis::use_mit_license("Civilytics Consulting LLC")'` + +This rewrites `LICENSE` to the two-line stub with the corrected holder and creates `LICENSE.md` with the full MIT text. It may also add `^LICENSE\.md$` to `.Rbuildignore` — that is correct and already present. + +- [ ] **Step 2: Verify** + +Run: `Rscript -e 'cat(readLines("LICENSE"), sep = "\n")'` +Expected: +``` +YEAR: 2026 +COPYRIGHT HOLDER: Civilytics Consulting LLC +``` + +Run: `Rscript -e 'cat(length(readLines("LICENSE.md")), "lines\n")'` +Expected: a non-zero count (the full MIT text, ~21 lines). + +- [ ] **Step 3: Confirm DESCRIPTION still declares the license correctly** + +Run: `Rscript -e 'cat(read.dcf("DESCRIPTION")[1, "License"], "\n")'` +Expected: `MIT + file LICENSE`. If `usethis` changed it, that is fine — leave whatever it wrote. + +- [ ] **Step 4: Commit** + +```bash +git add LICENSE LICENSE.md DESCRIPTION .Rbuildignore +git commit -m "chore: add full MIT text, name the copyright holder properly + +LICENSE held only the two-line stub and no LICENSE.md existed, so the +repo carried no license text for a human or for GitHub's detector." +``` + +--- + +## Task 6: Ship the vignettes + +`.Rbuildignore` excludes `^vignettes$`, so an installed package has no vignettes at all — while README tells users to run `vignette("total-spending", package = "uscogdata")`. Both vignettes build offline: `total-spending.Rmd` points `USCOGDATA_URL` at the bundled fixture, `population-denominators.Rmd` is `eval = FALSE`. + +**Files:** +- Modify: `.Rbuildignore` +- Test: `tests/testthat/test-config.R` (add) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/testthat/test-config.R`: + +```r +test_that("vignettes are not excluded from the build", { + skip_if_no_source_tree(".Rbuildignore") + ignore <- readLines(source_tree_path(".Rbuildignore"), warn = FALSE) + expect_false(any(grepl("^\\^vignettes\\$$", ignore))) + # The fixture is what lets R CMD check run offline with no credentials on + # r-universe and GitHub Actions. It must never be excluded. + expect_false(any(grepl("fixture_corpus", ignore, fixed = TRUE))) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: FAIL on the first expectation. + +- [ ] **Step 3: Remove the three lines** + +Delete these lines from `.Rbuildignore`: + +``` +^vignettes$ +^doc$ +^Meta$ +``` + +Leave every other line untouched — in particular `^_pkgdown\.yml$`, `^docs$`, `^data-raw$`, `^specs$`, `^plans$`, `^\.gitea$`, `^CLAUDE\.md$` are all correct exclusions. + +- [ ] **Step 4: Build the tarball and confirm the vignettes are in it** + +```bash +Rscript -e 'devtools::build(path = tempdir())' +``` +Then list the tarball contents: +```bash +tar -tzf "$(ls -t $(Rscript -e 'cat(tempdir())')/uscogdata_*.tar.gz | head -1)" | grep -E 'vignettes|inst/doc' +``` +Expected: both `.Rmd` files appear under `uscogdata/vignettes/`. + +- [ ] **Step 5: Run the tests** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add .Rbuildignore tests/testthat/test-config.R +git commit -m "fix: ship the vignettes + +.Rbuildignore excluded ^vignettes$, so vignette(\"total-spending\") failed +for every user -- while the README instructed them to run it. Both build +offline against the bundled fixture." +``` + +--- + +## Task 7: pkgdown reference index + +`_pkgdown.yml` lists 6 of 14 exports. pkgdown errors on topics missing from the index, so the docs site does not build. + +**Files:** +- Modify: `_pkgdown.yml` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/testthat/test-config.R`: + +```r +test_that("_pkgdown.yml indexes every exported topic", { + skip_if_no_source_tree("_pkgdown.yml", "NAMESPACE") + exports <- grep("^export\\(", readLines(source_tree_path("NAMESPACE"), warn = FALSE), value = TRUE) + exports <- sub("^export\\((.*)\\)$", "\\1", exports) + yml <- paste(readLines(source_tree_path("_pkgdown.yml"), warn = FALSE), collapse = "\n") + missing <- exports[!vapply(exports, function(e) grepl(paste0("\\b", e, "\\b"), yml), logical(1))] + expect_equal(missing, character(0)) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: FAIL listing the eight missing: `cog_categories`, `cog_explain`, `cog_find_peers`, `cog_geographic_rollup`, `cog_manifest`, `cog_mirror`, `cog_peer_compare`, `cog_recipes`. + +- [ ] **Step 3: Rewrite `_pkgdown.yml`** + +```yaml +url: https://civilytics.r-universe.dev/uscogdata + +template: + bootstrap: 5 + +reference: + - title: Financial data + desc: Spending, revenue and balance-sheet holdings for one or more governments. + contents: + - cog_spending + - cog_revenue + - cog_balances + - title: Search & basket + desc: Resolve place names into canonical govids. + contents: + - cog_gov_search + - cog_basket_resolution + - cog_basket_unresolved + - title: Comparison & aggregation + desc: Peer cohorts and geographic rollups. + contents: + - cog_find_peers + - cog_peer_compare + - cog_geographic_rollup + - title: Corpus metadata + desc: What the corpus contains, where it came from, and how to hold a local copy. + contents: + - cog_categories + - cog_recipes + - cog_manifest + - cog_explain + - cog_mirror + +articles: + - title: Concepts + navbar: ~ + contents: + - total-spending + - population-denominators +``` + +- [ ] **Step 4: Build the site** + +Run: `Rscript -e 'pkgdown::build_site(preview = FALSE)'` +Expected: completes without error. Any "Topics missing from index" warning means an export was missed. + +- [ ] **Step 5: Run the tests** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add _pkgdown.yml tests/testthat/test-config.R +git commit -m "docs: index all 14 exports in pkgdown, set the site url + +The reference index covered 6 of 14, so pkgdown errored on the missing +topics and the docs site did not build." +``` + +`docs/` is gitignored via `.Rbuildignore`/`.gitignore`; do not commit built output. + +--- + +## Task 8: README rewrite + +The current README addresses someone inside the repo tree: status reads "Under active development (Phase 2 of the cog_pipeline project)", it points at `../cog_pipeline/docs/reader-specification.md`, the install line is commented out, and developer/testing/release sections sit above anything a user needs. + +**Files:** +- Rewrite: `README.md` + +**Interfaces:** +- Consumes: `URL`/`BugReports` from Task 4, the default URL from Task 2, the measured latencies from Task 3 Step 4. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/testthat/test-config.R`: + +```r +test_that("README is written for a stranger, not a repo insider", { + skip_if_no_source_tree("README.md") + r <- paste(readLines(source_tree_path("README.md"), warn = FALSE), collapse = "\n") + + # No paths that only resolve inside Jared's checkout. + expect_false(grepl("../cog_pipeline", r, fixed = TRUE)) + # A real, uncommented install line. + expect_match(r, "install.packages", fixed = TRUE) + expect_false(grepl("# pak::pkg_install", r, fixed = TRUE)) + # The errata most likely to produce a plausible-looking wrong answer. + expect_match(r, "full US dollars", fixed = TRUE) + # The release-instructions section that conflicts with public CI is gone. + expect_false(grepl("Rbuildignore", r, fixed = TRUE)) + # Both read paths documented. + expect_match(r, "cog_mirror", fixed = TRUE) +}) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: FAIL. + +- [ ] **Step 3: Rewrite README.md in this order** + +Write these sections, in this sequence. Content requirements are exact; prose is yours. + +1. **Title + one-paragraph what-it-is.** Curated R reader over the Civilytics US Census of Governments finance corpus. State the coverage: government types 0–3 (state, county, municipality, township), **FY1967–FY2024** (no source data FY1968, FY1969), **46,148,034 rows**, **190.6 MB**. Add the r-universe version badge. + +2. **Install.** + ````markdown + ```r + install.packages("uscogdata", + repos = c("https://civilytics.r-universe.dev", + "https://cloud.r-project.org")) + ``` + Or from source: + ```r + pak::pkg_install("git::https://gitea.civilytics.org/Civilytics/uscogdata.git") + ``` + ```` + +3. **Quickstart — no configuration step.** Verbatim: + ````markdown + ```r + library(uscogdata) + + # Resolve a place name to a canonical government id + madison <- cog_gov_search(name = "Madison", state = "WI", type = 2) + + # Full spending history, real dollars, per capita + spend <- cog_spending(madison$canonical_govid[1]) + + # Every result carries its own provenance + cog_explain(spend) + ``` + ```` + +4. **Two ways to read the corpus.** Remote (default, zero setup) vs `cog_mirror()` (190 MB once). Include the measured table — **use the verb latencies recorded in Task 3 Step 4, not the raw-scan figures**: + + | | Remote (default) | Mirrored | + |---|---|---| + | Setup | none | `cog_mirror()`, 190.6 MB once | + | Disk | 0 MB — HTTP range requests | 190.6 MB | + | Per-query | network round trip | local | + | Right for | trying it out, teaching, one-off questions | repeated analysis, offline work, reproducibility | + + State plainly: the default reads from a public HuggingFace mirror, and **the escape hatch is one function call** — after `cog_mirror()`, no analysis touches an external service. + +5. **Amounts are in full US dollars.** Keep the existing text nearly verbatim — it is correct and carefully argued. Keep the `attr(r, "provenance")$transformations$units_conversion` example and the "do not multiply again" warning. + +6. **Concepts.** Condense the existing primary/direct/total and general/total revenue sections to ~1/3 their length, each ending with a pointer to `vignette("total-spending")`. Add a short **Reporting coverage** paragraph: the Census is a complete census only in years ending in 2 and 7; every other year is a sample; `provenance$coverage` reports `n_units_reporting` per year, and `coverage = "census"` / `"consistent"` control the mode. + +7. **How to cite.** `citation("uscogdata")` for the package; corpus is CC-BY-4.0, cite *Civilytics Consulting, US Census of Governments finance corpus*. + +8. **Contributing.** Two sentences plus a link to `CONTRIBUTING.md` (Task 10). + +**Delete outright:** the "Status" block, the `../cog_pipeline/...` reference, the entire "Developer notes / Testing / Releasing against the live corpus" section (it moves to `CONTRIBUTING.md`, minus the fixture-stripping advice, which is wrong and must not be carried over). + +- [ ] **Step 4: Run the quickstart verbatim in a clean session** + +```bash +Rscript -e ' + Sys.unsetenv("USCOGDATA_URL") + devtools::load_all(".", quiet = TRUE) + madison <- cog_gov_search(name = "Madison", state = "WI", type = 2) + spend <- cog_spending(madison$canonical_govid[1]) + cat("rows:", nrow(spend), "years:", paste(range(spend$year), collapse = "-"), "\n") + cog_explain(spend) +' +``` +Expected: runs clean with no configuration. If it errors, the README is wrong — fix the README, not the test. + +- [ ] **Step 5: Run the tests** + +Run: `Rscript -e 'devtools::test(filter = "config")'` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add README.md tests/testthat/test-config.R +git commit -m "docs: rewrite README for a stranger + +Reordered around a new user -- what it is, install, a quickstart that runs +with no configuration, then the dollars warning and the concepts. Drops +the sibling-repo path, the commented-out install line, and the +fixture-stripping release advice that conflicts with public CI." +``` + +--- + +## Task 9: NEWS.md rewrite + +The current NEWS is a pre-release churn log spanning the package's entire development (2026-04-23 to 2026-08-04, 140 commits), describing changes relative to states no user has seen. To a newcomer it reads as instability. + +**Files:** +- Rewrite: `NEWS.md` +- Modify: `README.md` and roxygen blocks receiving migrated content + +- [ ] **Step 1: Migrate the load-bearing content first** + +Before deleting anything, move each of these to its documentation home. Verify each lands before proceeding: + +| Content in current NEWS | Destination | +|---|---| +| Coverage disclosure — census vs sample years, the 597-vs-112 Wisconsin example, `coverage` modes | README §6 (Task 8) **and** `@details` in `R/rollup.R`'s roxygen for `cog_geographic_rollup()` | +| `complete = TRUE` — the `reported` / `census_zero` / `not_reported` table | `@details` in `R/spending.R` and `R/revenue.R` roxygen | +| Series breaks and `corpus_break_refs` | `@details` in `R/provenance.R`'s `cog_explain()` roxygen | +| Per-year F-33 population denominators | already in `vignette("population-denominators")` — verify, do not duplicate | + +Run `Rscript -e 'devtools::document()'` after editing roxygen. + +- [ ] **Step 2: Restructure NEWS.md** + +Three edits, in this order: + +1. **Prepend** the `0.3.0` section below. +2. **Keep** the existing `# uscogdata 0.2.0` section verbatim — it is a real + changelog (`"All Categories"`, the coverage-signposting fix, the + `n_units_reporting` documentation) and users deserve it. +3. **Delete** the entire `# uscogdata 0.1.0 (development)` section and + everything under it. That is pre-release churn; it stays in git. + +The new top section: + +```markdown +# uscogdata 0.3.0 + +First public release. + +`uscogdata` provides curated R verbs over the Civilytics US Census of +Governments finance corpus: unit-level financial profiles, geographic +rollups, and peer comparisons, with auditable provenance on every result. + +## What it covers + +Government types 0–3 (state, county, municipality, township), FY1967–FY2024 +(no source data for FY1968 or FY1969) — 46,148,034 rows across 56 fiscal +years. The corpus is published under CC-BY-4.0 and reads directly over +HTTPS, or locally after `cog_mirror()`. + +## The verbs + +`cog_spending()`, `cog_revenue()` and `cog_balances()` for flows and +holdings; `cog_gov_search()` to resolve place names (including basket mode +for many at once); `cog_find_peers()` and `cog_peer_compare()` for cohorts; +`cog_geographic_rollup()` for aggregates; `cog_categories()`, +`cog_recipes()`, `cog_manifest()` and `cog_explain()` for metadata and +provenance; `cog_mirror()` for a local copy. + +## Four things to know before your first query + +* **Amounts are in full US dollars.** The raw Census files report thousands; + the verbs multiply by 1000 on the way out. Do not multiply again. +* **Multi-government aggregates disclose their coverage.** The Census is a + complete census only in years ending in 2 and 7. Every result carries + `provenance$coverage` with per-year `n_units_reporting`. +* **Absence means two different things.** Before FY2012 an absent cell means + Census published $0; from FY2012 it means not reported. + `complete = TRUE` labels which. +* **Series breaks reach you unasked.** Catalogued breaks intersecting your + query appear in provenance and in `cog_explain()`. + +## Known limits + +* Special districts (type 4) and school districts (type 5) are out of scope. +* Per-capita rollups exclude governments with no F-33 population. +* Employee-retirement (`X`) codes stop at FY2016, when those systems moved + to the Annual Survey of Public Pensions. +``` + +- [ ] **Step 3: Verify no orphaned content** + +Run: `git show HEAD:NEWS.md > /tmp/news-old.md && wc -l /tmp/news-old.md NEWS.md` + +Read `/tmp/news-old.md` once more and confirm every substantive claim from the **deleted `0.1.0 (development)` section** either appears in the new `0.3.0` section, landed somewhere in Step 1, or is genuinely pre-release churn (version bumps, fixture regenerations, internal refactors). + +Then confirm the `0.2.0` section survived intact: + +```bash +diff <(git show HEAD:NEWS.md | sed -n '/^# uscogdata 0.2.0/,/^# uscogdata 0.1.0/p' | head -n -1) \ + <(sed -n '/^# uscogdata 0.2.0/,$p' NEWS.md) +``` +Expected: no output. Any diff means the `0.2.0` changelog was damaged — restore it. + +- [ ] **Step 4: Run the full suite** + +Run: `Rscript -e 'devtools::test()'` +Expected: PASS — `devtools::document()` in Step 1 regenerated `man/`, so this catches a malformed roxygen block. + +- [ ] **Step 5: Commit** + +```bash +git add NEWS.md README.md R/ man/ +git commit -m "docs: recast NEWS around the first public release + +The changelog described changes relative to states no user ever saw, which +reads as instability to someone deciding whether to depend on this. The +load-bearing caveats move into README and roxygen, where they belong; the +pre-release history stays in git." +``` + +--- + +## Task 10: CONTRIBUTING.md + +**Files:** +- Create: `CONTRIBUTING.md` +- Modify: `.Rbuildignore` + +- [ ] **Step 1: Write CONTRIBUTING.md** + +It must contain, in this order: + +1. **Canonical source note.** Development happens on `gitea.civilytics.org/Civilytics/uscogdata`; `github.com/civilytics/uscogdata` is a mirror that accepts issues and pull requests. + +2. **What happens to a GitHub PR.** Verbatim explanation: it is fetched and landed on the canonical repo, then closes itself as merged when the mirror syncs — because the maintainer merges with `--no-ff`, preserving the contributor's commits and SHAs. Say plainly that a PR closing without a "Merged by" click is normal and not a rejection. + +3. **Running the tests.** + ````markdown + ```r + devtools::test() # uses the bundled fixture; no network, no credentials + ``` + ```` + Note that `tests/testthat/setup.R` points `USCOGDATA_URL` at + `inst/extdata/fixture_corpus/` automatically. + +4. **Testing against the live corpus.** + ````markdown + ```r + USCOGDATA_LIVE_TEST=true devtools::test(filter = "live-corpus") + ``` + ```` + Explain why it exists: every other test path uses a local corpus, which is + how the remote-read defect fixed for 0.3.0 went unnoticed. + +5. **Do not exclude the fixture from the build.** State the reason — it is what lets `R CMD check` pass on r-universe and GitHub Actions with no credentials. + +6. **Release checklist**, moved from README: run the suite against both the fixture and the live corpus, `pkgdown::build_site()`, `R CMD check --as-cran`, tag, then update the r-universe registry pin. + +**Do not carry over** the README's instruction to add `^inst/extdata/fixture_corpus$` to `.Rbuildignore`. It is wrong. + +- [ ] **Step 2: Exclude it from the build** + +Add to `.Rbuildignore`: + +``` +^CONTRIBUTING\.md$ +``` + +- [ ] **Step 3: Verify the build is clean** + +Run: `Rscript -e 'devtools::check(document = FALSE, args = "--no-manual")'` +Expected: 0 errors, 0 warnings. Notes about package size (the 15 MB fixture) are expected and acceptable — this package is not going to CRAN. + +- [ ] **Step 4: Commit** + +```bash +git add CONTRIBUTING.md .Rbuildignore +git commit -m "docs: add CONTRIBUTING with the canonical-on-Gitea PR flow + +Moves developer and release instructions out of the README, minus the +fixture-stripping advice, which would break the vignette and leave public +CI unable to check without credentials." +``` + +--- + +## Final verification + +Run before declaring the release ready. Every one of these must pass. + +- [ ] **1. Full suite, offline, no credentials** + `Rscript -e 'devtools::test()'` — the property public CI depends on. + +- [ ] **2. Live corpus** + `USCOGDATA_LIVE_TEST=true Rscript -e 'devtools::test()'` + +- [ ] **3. `R CMD check --as-cran`** + `Rscript -e 'devtools::check(args = "--as-cran")'` — 0 errors, 0 warnings. + +- [ ] **4. pkgdown** + `Rscript -e 'pkgdown::build_site(preview = FALSE)'` + +- [ ] **5. Vignettes resolve from an installed copy** + ```bash + Rscript -e 'devtools::install(build_vignettes = TRUE, quiet = TRUE)' + Rscript -e 'v <- vignette("total-spending", package = "uscogdata"); stopifnot(nzchar(v$File)); cat("OK\n")' + ``` + +- [ ] **6. Cold-start check.** On a machine (or container) that has never had this package: install it, then run the README quickstart **verbatim with no environment variables set**. + ```bash + docker run --rm -v "$PWD":/pkg rocker/r-ver:4.4 bash -c ' + apt-get update -qq && apt-get install -y -qq libcurl4-openssl-dev libssl-dev >/dev/null + Rscript -e "install.packages(c(\"pak\"), repos=\"https://cloud.r-project.org\")" \ + -e "pak::pkg_install(\"local::/pkg\")" \ + -e "library(uscogdata); m <- cog_gov_search(name=\"Madison\", state=\"WI\", type=2); s <- cog_spending(m\$canonical_govid[1]); cat(\"rows:\", nrow(s), \"\n\")" + ' + ``` + **This is the only check that catches the P0 class of fault**, and its absence is why the fault survived. Do not skip it. + +- [ ] **7. Verb latency re-measured** against the live corpus, and the README table updated if the numbers moved from what Task 3 Step 4 recorded. + +## Out of scope for this plan + +Flipping the Gitea repo public, `gitleaks`, the GitHub mirror and its Actions matrix, the Gitea push workflow, the r-universe registry, and tagging `v0.3.0`. Those follow after this plan's final verification is green — r-universe publishes check results on registration, so registering before checks pass means a red badge on day one. Corrections intake and announcement posts are deferred by decision (see the spec). diff --git a/specs/2026-08-08-public-release-design.md b/specs/2026-08-08-public-release-design.md new file mode 100644 index 0000000..90c0c14 --- /dev/null +++ b/specs/2026-08-08-public-release-design.md @@ -0,0 +1,296 @@ +# `uscogdata` 0.3.0 — public release + +**Date:** 2026-08-08 · **Status:** design, awaiting approval +**Scope:** release-readiness, README, NEWS. Distribution mechanics recorded here as +decided, sequenced after the package is clean. + +`uscogdata` is feature-complete and the corpus it reads has been public on +HuggingFace since 2026-08-07 (294 downloads as of this writing). The API built on +it is live. What does not exist is a public *package*: the repo is private, there +is no install path, and — measured, not assumed — **a stranger who installed it +today could not read the corpus at all.** + +This spec covers making that untrue. + +## Decisions locked + +| Decision | Choice | +|---|---| +| Canonical source | `gitea.civilytics.org/Civilytics/uscogdata`, flipped public | +| Public mirror | `github.com/civilytics/uscogdata` — issues, PRs, multi-OS check, CDN | +| Mirror mechanism | Gitea Actions non-force `git push` (not a push mirror) | +| Binaries | `civilytics.r-universe.dev`, registry pinned to a release tag | +| Author of record | Jared E. Knowles ``, ORCID `0000-0003-0005-9478` | +| Copyright | Civilytics Consulting LLC (`cph`, `fnd`) | +| License | MIT (package) · CC-BY-4.0 (corpus) | +| Corrections intake | Deferred — see *Out of scope* | +| Other packages | Parked until this one walks the path end to end | + +## P0 — the corpus is unreachable + +Two independent faults, either of which alone is fatal. + +**No corpus URL exists.** `R/config.R` defaults to the literal +`REPLACE_WITH_SHARE_TOKEN` sentinel, and no file in the repo supplies a working +one. A new user calling any verb gets `uscogdata_url_not_configured` with no path +to resolution. + +**Remote reads are broken regardless.** Every partitioned view globs: + +```sql +FROM read_parquet('{url}data/long/**/*.parquet', hive_partitioning = true) +``` + +DuckDB 1.5.5 refuses globs over generic HTTP. Its suggested +`allow_asterisks_in_http_paths` escape hatch does not help — it forwards the +literal `**/*` as a filename and 404s, because plain HTTP exposes no directory +listing to expand against. + +The package therefore works only against a **local path**. That is how the API +runs it (`CORPUS_HOST_PATH` is a host mount on maxwell) and how the tests run +(bundled fixture), which is why the fault went unnoticed. The README's headline +claim — *"Reads the published corpus directly from Nextcloud via DuckDB httpfs — +no local bulk downloads required"* — is currently false. + +### Fix: enumerate from the manifest, do not glob + +`manifest.json` already lists every partition under `files.long_partitions[]` +with `path`, `year`, `sha256`, `row_count` and `size_bytes` — 56 of them. +Substituting an explicit file list for the glob was measured against the +published corpus on 2026-08-08: + +| Path | Result | +|---|---| +| `https://…/data/long/**/*.parquet` (default) | error — globs unsupported over HTTP | +| same, `allow_asterisks_in_http_paths = true` | error — literal `**/*` 404s | +| `hf://datasets/civilytics/us-cog-finance/…` glob | 46,148,034 rows | +| **explicit list over plain https** | **46,148,034 rows** | + +`hive_partitioning = true` still recovers `year` from the paths under +enumeration, so no downstream view or verb changes. + +Enumeration is preferred over `hf://` deliberately. It is **host-agnostic** — +Nextcloud, HuggingFace, or any static server take the same code path — where +`hf://` would tie the default to one vendor's protocol and still need +special-casing, since manifest fetching goes through `httr2`, which cannot speak +`hf://`. Enumeration also *removes* a dependency (globbing) rather than adding +one, and the manifest's per-file `sha256` becomes available for integrity +checking later. + +Views are registered from `inst/sql/` with `{url}` substitution in +`R/views.R:.register_views()`. The list must be built once per session from the +already-fetched manifest and substituted the same way, so the change is confined +to view registration and does not touch verb code. + +### Fix: ship a working default + +`R/config.R`'s default becomes the public HuggingFace `resolve/main/` URL: +CC-BY-4.0, no token to publish, CDN-backed, and it keeps maxwell's uplink out of +the path — the same reasoning behind the GitHub mirror and r-universe. + +This means `library(uscogdata)` followed by a verb works with **zero +configuration**, which is what makes the package demonstrable in a README and +later in a post. `USCOGDATA_URL` and `options(uscogdata.url=)` continue to +override, so the Nextcloud copy and local mirrors are unaffected. + +The `uscogdata_url_not_configured` error class stays — it still fires for an +explicitly-set empty or placeholder URL — but ceases to be the default +experience. + +### Consequence: `cog_mirror()` is promoted + +Measured cost of the remote default, from efron on a good connection: + +| | | +|---|---| +| Whole corpus | **190.6 MB**, 56 partitions, 46,148,034 rows, FY1967–FY2024 | +| One government, one year | 1.5 s | +| One government, all 56 years | 2.8 s | +| Disk written | **0.00 MB** — range requests only; `external_file_cache` is in-memory | + +Nothing persists locally beyond the shared `httpfs` extension in `~/.duckdb` (a +few MB, once per machine, across all DuckDB use). Costs are RAM and per-query +bandwidth, since nothing caches between sessions. + +Those timings are raw scans. Real verbs additionally join crosswalks, resolve +categories and assemble provenance, so end-to-end verb latency will be higher and +**must be re-measured once the fix lands** — it cannot be measured today. + +The corpus being only 190.6 MB makes `cog_mirror()` a first-class option rather +than a developer footnote. The README presents **both paths**: + +- **Remote (default, zero setup)** — trying it out, teaching, one-off questions. +- **Mirrored (`cog_mirror()`, 190 MB once)** — repeated or heavy analysis, + offline work, reproducibility, or preferring not to depend on HuggingFace. + +The second is also the honest answer to the vendor-dependency question raised by +defaulting to HuggingFace: **the escape hatch is one function call and 190 MB**, +after which no analysis touches an external service. The README says so +explicitly. That is the difference between a convenience default and lock-in. + +## Release-readiness fixes + +| # | Issue | Fix | +|---|---|---| +| 1 | `MaxCorpusSchema: 5` in DESCRIPTION; `.validate_schema()` accepts `4,5,6,7`; published corpus is **7** | `MaxCorpusSchema: 7` | +| 2 | `^vignettes$` in `.Rbuildignore` — both vignettes absent from the installed package, while README tells users to run `vignette("total-spending")` | Remove `^vignettes$`, `^doc$`, `^Meta$`. Both vignettes build offline (`total-spending` reads the bundled fixture; `population-denominators` is `eval = FALSE`) | +| 3 | `_pkgdown.yml` reference index covers 6 of 14 exports — pkgdown errors on missing topics | Add `cog_categories`, `cog_explain`, `cog_find_peers`, `cog_geographic_rollup`, `cog_manifest`, `cog_mirror`, `cog_peer_compare`, `cog_recipes`; set `url:` | +| 4 | No `URL:` / `BugReports:` in DESCRIPTION | Add both, pointing at the GitHub mirror | +| 5 | No `LICENSE.md`; `LICENSE` holder reads `Civilytics` | `usethis::use_mit_license("Civilytics Consulting LLC")` | +| 6 | README instructs stripping the fixture at release | Delete that section — see below | +| 7 | `Authors@R` is an org with no human | Jared E. Knowles `aut`/`cre` + ORCID; Civilytics Consulting LLC `cph`/`fnd` | + +**On #6.** The advice to add `^inst/extdata/fixture_corpus$` to `.Rbuildignore` +is CRAN-sized thinking (5 MB limit) and this package is not going to CRAN. +Stripping the 15 MB fixture would break `total-spending.Rmd`, which reads from +it, and would leave r-universe and GitHub Actions unable to run the 28 test files +without a corpus credential. **The fixture is what lets `R CMD check` pass +anywhere with zero secrets** — precisely what public CI needs. It ships. + +## README + +The current README addresses someone standing inside the repo tree: status reads +"Under active development (Phase 2 of the cog_pipeline project)", it points at +`../cog_pipeline/docs/reader-specification.md`, the install line is commented +out, and developer, testing and release sections sit above anything a user needs. + +Restructured around a stranger, in this order: + +1. **What this is** — one paragraph, and what the corpus covers (types 0–3, + FY1967–FY2024, 46M rows, 190.6 MB). +2. **Install** — r-universe first (binaries), git second. +3. **Quickstart that actually runs** — resolve a government, get its history, + print provenance. No configuration step. +4. **Two ways to read the corpus** — remote default vs `cog_mirror()`, with the + measured numbers and the independence note. +5. **Amounts are in full US dollars** — kept near the top. This is the errata + most likely to produce a wrong answer that looks plausible. +6. **Concepts** — primary/direct/total spending, general/total revenue, + coverage. Condensed, linking to the vignettes for the full treatment. +7. **How to cite** — `citation("uscogdata")`, corpus CC-BY-4.0 attribution. +8. **Contributing** — canonical-on-Gitea PR flow. + +Developer notes, testing instructions and release procedure move to +`CONTRIBUTING.md`. Every path reference to a sibling repo is removed or replaced +with a URL that resolves for someone who has only this repo. + +## NEWS.md + +`NEWS.md` currently holds two sections. `0.2.0` is a legitimate changelog — the +`"All Categories"` reserved value, the coverage-signposting fix, the +`n_units_reporting` documentation — and it stays. Beneath it, +`0.1.0 (development)` is a pre-release churn log: changes described relative to +states no user has ever seen ("Breaking: corpus schema_version 4", "the package +now requires…"), spanning the package's entire pre-release development. To a +newcomer deciding whether to depend on this, that section reads as instability. + +**A new `0.3.0` section is added at the top, framed as the first public +release**: what the package does, what the corpus covers, and the caveats that +are genuinely load-bearing. **`0.2.0` is kept verbatim.** **`0.1.0 (development)` +is dropped** — that history stays in git, where it belongs. + +The version is `0.3.0` rather than `0.2.0` because this release changes +user-visible behaviour: remote corpus reads go from broken to working, and the +default URL from a dead placeholder to a live corpus. It is also not `1.0.0` — +the corpus still excludes government types 4 and 5 pending validation, so a +stability promise would overclaim. No git tag exists for any prior version; +`chore: release 0.2.0` bumped `DESCRIPTION` and `NEWS` only. + +The substantive content is migrated, not deleted. These are hard-won and belong +in documentation rather than buried in a changelog: + +| Content | Destination | +|---|---| +| Coverage disclosure on multi-government aggregates (census vs sample years) | README concepts + `cog_geographic_rollup()` docs | +| `complete = TRUE` three-way absence semantics (`reported` / `census_zero` / `not_reported`) | `cog_spending()` / `cog_revenue()` docs | +| Series-break and corpus-break surfacing | README + `cog_explain()` docs | +| $1,000s → full dollars conversion | README, already prominent | +| Per-year F-33 population denominators | `population-denominators` vignette, already there | + +This also makes NEWS reusable as raw material for the release announcement, +which is the stated downstream purpose. + +## Distribution mechanics + +Recorded as decided; executed after the package is clean and checks are green. + +**Sequence matters.** r-universe publishes check results the moment a package is +registered. Registering before the fixes above land means a red badge on day one, +which is a worse first impression than a week's delay. + +1. `gitleaks` over full history. A coarse grep found nothing across 140 commits + and the default corpus URL is still the placeholder sentinel, but a proper + scan is the gate on an irreversible action. +2. Flip the Gitea repo public. Disable Gitea issues on it, so there is exactly + one inbox. +3. Create `github.com/civilytics/uscogdata`. Add `.github/workflows/` for the + Windows/macOS/Linux `R CMD check` matrix — the platforms the Gitea runner + cannot provide, and which this package has never been tested on despite + depending on duckdb and httr2. Gitea reads `.gitea/workflows`, GitHub reads + `.github/workflows`; both live in one tree without colliding. +4. Gitea Actions workflow pushing to GitHub **without `--force`**, so divergence + fails loudly in CI rather than silently overwriting. +5. Add `jared@civilytics.com` as a verified secondary email on the GitHub + account — r-universe links maintainer identity by matching DESCRIPTION's email + against registered GitHub emails, and the association only takes effect on the + next build. +6. Tag `v0.3.0`. Create `github.com/civilytics/civilytics.r-universe.dev` with a + `packages.json` pinned to the tag, pointing at the GitHub mirror rather than + Gitea so clone traffic stays off maxwell. Install the r-universe app. + +### PR flow + +Never press Merge on GitHub. A merge there is overwritten by the next sync, the +PR still displays "Merged", and nothing says otherwise. + +```sh +git remote add github https://github.com/civilytics/uscogdata.git +git config --add remote.github.fetch '+refs/pull/*/head:refs/remotes/github/pr/*' +git fetch github +git switch -c pr-42 github/pr/42 # test +git switch main && git merge --no-ff pr-42 +git push origin main # Gitea -> mirror -> GitHub +``` + +GitHub auto-closes a PR as merged once its head commit becomes an ancestor of the +base branch, so `--no-ff` — which preserves the contributor's SHAs — makes the PR +close itself when the mirror pushes. **For external PRs, merge; do not squash or +rebase.** Squashing rewrites the SHAs, the auto-close never fires, and closing by +hand reads to a first-time contributor as rejection. + +`CONTRIBUTING.md` states this, and a GitHub Action comments it on incoming PRs. +No CLA; no DCO. + +## Verification + +The release is not done until all of these pass: + +1. `R CMD check --as-cran` clean on Linux, and on Windows and macOS via the + GitHub matrix. This package has never been checked on the latter two. +2. Full test suite (28 files) green against the **bundled fixture**, offline, + with no credentials — the property public CI depends on. +3. Full test suite green against the **live corpus**, which additionally + exercises the enumeration fix that the fixture's local path cannot. +4. `pkgdown::build_site()` completes. +5. Both vignettes present in the built tarball and + `vignette("total-spending", package = "uscogdata")` resolves from an + installed copy. +6. **Cold-start check on a machine that has never seen this package:** install + from r-universe, `library(uscogdata)`, run the README quickstart verbatim with + no environment variables set. This is the only test that catches the P0 class + of fault, and its absence is why the fault survived. +7. End-to-end verb latency re-measured against the live corpus and the README's + numbers updated if they moved. + +## Out of scope + +- **Corrections intake.** Deferred by decision. Consequence: the release cannot + invite data-error reports or make the "traceable and correctable" claim that + most distinguishes this corpus from Census's own files. `BugReports:` points at + package issues only. A verified correction should eventually terminate as a + `lineage_event` or `series_break` row so it propagates through provenance to + every consumer — that design is unstarted. +- **Announcement posts.** Deferred. The API announcement is gated on corrections + landing and merits a Civic Pulse edition. +- **The rest of the R package backlog.** Parked until this one completes the path. +- **`cog_pipeline` publication.** Stays private. diff --git a/tests/testthat/test-balances.R b/tests/testthat/test-balances.R index 86d5ba4..c278a73 100644 --- a/tests/testthat/test-balances.R +++ b/tests/testthat/test-balances.R @@ -66,7 +66,7 @@ test_that("inst/sql/26-balance_long.sql enforces NOT is_aggregate (real SQL text sql_dir <- system.file("sql", package = "uscogdata") .read_view_sql <- function(filename) { txt <- paste(readLines(file.path(sql_dir, filename), warn = FALSE), collapse = "\n") - gsub("\\{url\\}", paste0(tmp, "/"), txt, fixed = FALSE) + uscogdata:::.render_view_sql(txt, paste0(tmp, "/")) } con <- DBI::dbConnect(duckdb::duckdb()) diff --git a/tests/testthat/test-config.R b/tests/testthat/test-config.R index 3461ab2..cdf3580 100644 --- a/tests/testthat/test-config.R +++ b/tests/testthat/test-config.R @@ -64,3 +64,109 @@ test_that(".resolve_url does not invent a slash for an empty setting", { withr::local_options(uscogdata.url = "") expect_equal(.resolve_url(), "") }) + +test_that("the default corpus URL is real, not a placeholder", { + # setup.R points USCOGDATA_URL at the bundled fixture for the whole suite, + # so both the env var and the option have to be cleared to see the default. + withr::local_envvar(USCOGDATA_URL = NA) + withr::local_options(uscogdata.url = NULL) + url <- .resolve_url() + expect_false(grepl("REPLACE_WITH", url, fixed = TRUE)) + expect_match(url, "^https://") + expect_match(url, "/$") +}) + +test_that("an explicitly-set sentinel URL still aborts", { + # The guard must survive the default change: a user who half-edited a + # copied config still gets the actionable error. + withr::local_envvar( + USCOGDATA_URL = "https://other.example/s/REPLACE_WITH_SHARE_TOKEN/x/" + ) + expect_error( + .check_url_configured(.resolve_url()), + class = "uscogdata_url_not_configured" + ) +}) + +test_that("DESCRIPTION carries release metadata", { + skip_if_no_source_tree("DESCRIPTION") + d <- read.dcf(source_tree_path("DESCRIPTION")) + fields <- colnames(d) + + expect_true(all(c("URL", "BugReports") %in% fields)) + expect_match(d[1, "Authors@R"], "Knowles", fixed = TRUE) + expect_match(d[1, "Authors@R"], "0000-0003-0005-9478", fixed = TRUE) + expect_match(d[1, "Authors@R"], "Civilytics Consulting LLC", fixed = TRUE) + + # The gate in .validate_schema() accepts up to 7 and the published corpus + # IS 7; DESCRIPTION must not claim otherwise. + expect_equal(as.integer(d[1, "MaxCorpusSchema"]), 7L) + + # Authors@R must actually parse -- a malformed person() call is only + # caught at citation()/build time otherwise. + people <- eval(parse(text = d[1, "Authors@R"])) + expect_s3_class(people, "person") + expect_true("cre" %in% unlist(lapply(people, function(p) p$role))) +}) + +test_that("LICENSE and LICENSE.md name the same copyright holder", { + skip_if_no_source_tree("LICENSE", "LICENSE.md") + holder <- sub("^COPYRIGHT HOLDER:\\s*", "", + grep("^COPYRIGHT HOLDER:", readLines(source_tree_path("LICENSE"), + warn = FALSE), value = TRUE)) + full <- paste(readLines(source_tree_path("LICENSE.md"), warn = FALSE), collapse = "\n") + + expect_equal(holder, "Civilytics Consulting LLC") + expect_match(full, holder, fixed = TRUE) + # usethis::use_mit_license() writes LICENSE.md but leaves an existing + # LICENSE alone, which is how the two came to disagree in the first place. + expect_match(full, "MIT License", fixed = TRUE) +}) + +test_that("vignettes are not excluded from the build", { + skip_if_no_source_tree(".Rbuildignore") + ignore <- readLines(source_tree_path(".Rbuildignore"), warn = FALSE) + expect_false(any(grepl("^\\^vignettes\\$$", ignore))) + # The fixture is what lets R CMD check run offline with no credentials on + # r-universe and GitHub Actions. It must never be excluded. + expect_false(any(grepl("fixture_corpus", ignore, fixed = TRUE))) + # doc/ and Meta/ ARE build artefacts of devtools::build_vignettes() and must + # stay excluded -- R CMD build regenerates inst/doc/ from vignettes/ on its + # own, and leaving them in earns a "non-standard file at top level" NOTE. + expect_true(any(grepl("^\\^doc\\$$", ignore))) + expect_true(any(grepl("^\\^Meta\\$$", ignore))) +}) + +test_that("_pkgdown.yml indexes every exported topic", { + skip_if_no_source_tree("_pkgdown.yml", "NAMESPACE") + exports <- grep("^export\\(", readLines(source_tree_path("NAMESPACE"), warn = FALSE), + value = TRUE) + exports <- sub("^export\\((.*)\\)$", "\\1", exports) + yml <- paste(readLines(source_tree_path("_pkgdown.yml"), warn = FALSE), collapse = "\n") + missing <- exports[!vapply(exports, + function(e) grepl(paste0("\\b", e, "\\b"), yml), + logical(1))] + # pkgdown errors on topics missing from the index, so an unlisted export + # means the docs site does not build at all. + expect_equal(missing, character(0)) +}) + +test_that("README is written for a stranger, not a repo insider", { + skip_if_no_source_tree("README.md") + r <- paste(readLines(source_tree_path("README.md"), warn = FALSE), collapse = "\n") + + # No paths that only resolve inside a maintainer's checkout. + expect_false(grepl("../cog_pipeline", r, fixed = TRUE)) + # A real, uncommented install line. + expect_match(r, "install.packages", fixed = TRUE) + expect_false(grepl("# pak::pkg_install", r, fixed = TRUE)) + # The errata most likely to produce a plausible-looking wrong answer. + expect_match(r, "full US dollars", fixed = TRUE) + # The release advice that conflicts with public CI is gone. + expect_false(grepl("Rbuildignore", r, fixed = TRUE)) + # Both read paths documented. + expect_match(r, "cog_mirror", fixed = TRUE) + # cog_spending() has no default for `years`; a quickstart that omits it + # errors on the reader's first call. + expect_match(r, "years\\s*=", perl = TRUE) +}) diff --git a/tests/testthat/test-live-corpus.R b/tests/testthat/test-live-corpus.R new file mode 100644 index 0000000..ca9bd86 --- /dev/null +++ b/tests/testthat/test-live-corpus.R @@ -0,0 +1,62 @@ +# Network-gated. Set USCOGDATA_LIVE_TEST=true to run. +# +# This file exists because the defect fixed for 0.3.0 -- no remote corpus was +# readable at all, because DuckDB cannot expand a glob over generic HTTP -- +# survived precisely because every other test path used a LOCAL corpus (the +# bundled fixture), and so did the API in production (a host mount). Nothing +# ever exercised the package the way a new user does. +skip_live <- function() { + testthat::skip_if_not( + identical(tolower(Sys.getenv("USCOGDATA_LIVE_TEST", "")), "true"), + "live-corpus test: set USCOGDATA_LIVE_TEST=true to run" + ) +} + +# The suite's setup.R pins USCOGDATA_URL to the bundled fixture, so reaching +# the default requires clearing both the env var and the option. +with_default_corpus <- function(code) { + withr::local_envvar( + USCOGDATA_URL = NA, USCOGDATA_FIXTURE_URL = NA, + .local_envir = parent.frame() + ) + withr::local_options(uscogdata.url = NULL, .local_envir = parent.frame()) + cog_close() + withr::defer(cog_close(), envir = parent.frame()) + force(code) +} + +test_that("the package reads the public corpus with no configuration at all", { + skip_live() + with_default_corpus({ + g <- cog_gov_search(name = "Madison", state = "WI", type = 2) + expect_gt(nrow(g), 0) + + s <- cog_spending(g$canonical_govid[1], years = 2022) + expect_gt(nrow(s), 0) + expect_true(all(c("amt_nominal", "year", "category") %in% names(s))) + + # Amounts are full dollars, already x1000. A city's annual spending is + # millions, not thousands -- this catches a regression that dropped or + # doubled the conversion. + expect_gt(sum(s$amt_nominal, na.rm = TRUE), 1e6) + + p <- attr(s, "provenance") + expect_true(isTRUE(p$transformations$units_conversion$applied)) + expect_equal(p$transformations$units_conversion$multiplier, 1000) + }) +}) + +test_that("a multi-decade query reads across many partitions", { + skip_live() + with_default_corpus({ + g <- cog_gov_search(name = "Madison", state = "WI", type = 2) + # `years` is required on cog_spending() -- there is no full-history + # default at the reader level (the API's /profile route supplies one). + s <- cog_spending(g$canonical_govid[1], years = 2000:2022) + # Enumeration builds one read_parquet() path per requested partition. If + # the list were truncated, or silently collapsed to a single file, the + # returned span is what catches it. + expect_gt(diff(range(s$year)), 10) + expect_gt(length(unique(s$year)), 5) + }) +}) diff --git a/tests/testthat/test-long-files.R b/tests/testthat/test-long-files.R new file mode 100644 index 0000000..af84007 --- /dev/null +++ b/tests/testthat/test-long-files.R @@ -0,0 +1,64 @@ +test_that(".long_files_sql enumerates every partition the manifest lists", { + manifest <- list(files = list(long_partitions = list( + list(year = 2011L, path = "data/long/year=2011/part-0.parquet"), + list(year = 2012L, path = "data/long/year=2012/part-0.parquet") + ))) + expect_equal( + uscogdata:::.long_files_sql("https://example.org/corpus/", manifest), + paste0( + "['https://example.org/corpus/data/long/year=2011/part-0.parquet',", + "'https://example.org/corpus/data/long/year=2012/part-0.parquet']" + ) + ) +}) + +test_that(".long_files_sql falls back to the glob when no partition list is present", { + # test-views.R registers views with a hand-built manifest that has no + # `files` element. That must keep working: the glob is valid for the + # local paths such a manifest is used with. + expect_equal( + uscogdata:::.long_files_sql("/tmp/corpus/", list(schema_version = 4L)), + "'/tmp/corpus/data/long/**/*.parquet'" + ) + expect_equal( + uscogdata:::.long_files_sql("/tmp/corpus/", list(files = list(long_partitions = list()))), + "'/tmp/corpus/data/long/**/*.parquet'" + ) +}) + +test_that("the enumerated list matches the bundled fixture's partition count", { + skip_if_no_corpus() + m <- jsonlite::fromJSON( + file.path(fixture_corpus_path(), "manifest.json"), simplifyVector = FALSE + ) + out <- uscogdata:::.long_files_sql(fixture_corpus_path(), m) + expect_equal( + lengths(regmatches(out, gregexpr("part-0\\.parquet", out))), + length(m$files$long_partitions) + ) +}) + +test_that("no view SQL survives rendering with an unsubstituted token", { + # Introducing {long_files} broke four test sites that had hand-rolled the + # {url} substitution -- each failed with a DuckDB parser error on the + # surviving brace. This asserts the whole SQL directory renders clean, so + # a future token cannot reintroduce that silently. + sql_dir <- system.file("sql", package = "uscogdata") + for (f in list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE)) { + rendered <- uscogdata:::.render_view_sql( + paste(readLines(f, warn = FALSE), collapse = "\n"), "/tmp/corpus/" + ) + expect_false(grepl("\\{[a-z_]+\\}", rendered), label = basename(f)) + } +}) + +test_that("registered `long` view reads through the enumerated list", { + skip_if_no_corpus() + with_fixture_corpus({ + con <- uscogdata:::.ensure_session() + n <- DBI::dbGetQuery(con, "SELECT count(*) AS n FROM long")$n + expect_gt(n, 0) + yrs <- DBI::dbGetQuery(con, "SELECT DISTINCT year FROM long ORDER BY year")$year + expect_true(all(c(2011, 2012, 2019, 2020) %in% yrs)) + }) +}) diff --git a/tests/testthat/test-manifest.R b/tests/testthat/test-manifest.R index 31c9462..f2b3937 100644 --- a/tests/testthat/test-manifest.R +++ b/tests/testthat/test-manifest.R @@ -4,12 +4,14 @@ # protect users from silent failures when USCOGDATA_URL is misconfigured # or returns non-JSON content. -test_that("cog_open aborts with actionable error when URL is the placeholder default", { +test_that("cog_open aborts with actionable error when URL contains the sentinel", { uscogdata:::cog_close() on.exit(uscogdata:::cog_close(), add = TRUE) - placeholder <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/" - withr::with_envvar(c(USCOGDATA_URL = placeholder), { + # No longer the package default (that is the public HF corpus). This is a + # user who copied a config template and did not finish editing it. + sentinel_url <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/" + withr::with_envvar(c(USCOGDATA_URL = sentinel_url), { expect_error( uscogdata:::cog_open(), class = "uscogdata_url_not_configured" @@ -35,8 +37,10 @@ test_that("placeholder guard error names both env var and option as remediation" uscogdata:::cog_close() on.exit(uscogdata:::cog_close(), add = TRUE) - placeholder <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/" - withr::with_envvar(c(USCOGDATA_URL = placeholder), { + # No longer the package default (that is the public HF corpus). This is a + # user who copied a config template and did not finish editing it. + sentinel_url <- "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/" + withr::with_envvar(c(USCOGDATA_URL = sentinel_url), { msg <- tryCatch(uscogdata:::cog_open(), error = conditionMessage) expect_match(msg, "USCOGDATA_URL", fixed = TRUE) expect_match(msg, "uscogdata.url", fixed = TRUE) diff --git a/tests/testthat/test-views.R b/tests/testthat/test-views.R index 654c744..08c548c 100644 --- a/tests/testthat/test-views.R +++ b/tests/testthat/test-views.R @@ -103,7 +103,7 @@ test_that("inst/sql/22- and 23- harmonized views enforce every WHERE predicate ( sql_dir <- system.file("sql", package = "uscogdata") .read_view_sql <- function(filename) { txt <- paste(readLines(file.path(sql_dir, filename), warn = FALSE), collapse = "\n") - gsub("\\{url\\}", paste0(tmp, "/"), txt, fixed = FALSE) + uscogdata:::.render_view_sql(txt, paste0(tmp, "/")) } con <- DBI::dbConnect(duckdb::duckdb()) @@ -183,7 +183,7 @@ test_that("inst/sql/24- and 25- IG views retain aggregates, COALESCE NULL harmon sql_dir <- system.file("sql", package = "uscogdata") .read_view_sql <- function(filename) { txt <- paste(readLines(file.path(sql_dir, filename), warn = FALSE), collapse = "\n") - gsub("\\{url\\}", paste0(tmp, "/"), txt, fixed = FALSE) + uscogdata:::.render_view_sql(txt, paste0(tmp, "/")) } con <- DBI::dbConnect(duckdb::duckdb()) @@ -335,7 +335,7 @@ test_that(".harmonization_view_files guard is necessary: registration against a sql_dir <- system.file("sql", package = "uscogdata") .read_view_sql <- function(filename) { txt <- paste(readLines(file.path(sql_dir, filename), warn = FALSE), collapse = "\n") - gsub("\\{url\\}", url, txt, fixed = FALSE) + uscogdata:::.render_view_sql(txt, url) } con2 <- DBI::dbConnect(duckdb::duckdb()) on.exit(DBI::dbDisconnect(con2, shutdown = TRUE), add = TRUE)