From f7035049e760c7a592f5b087c9386b6852ec60dc Mon Sep 17 00:00:00 2001
From: Jared Knowles <jared@civilytics.com>
Date: Thu, 23 Apr 2026 09:05:54 -0400
Subject: [PATCH] =?UTF-8?q?feat:=20package=20skeleton=20=E2=80=94=20DESCRI?=
 =?UTF-8?q?PTION,=20NAMESPACE,=20session/manifest/cache/views?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Minimal skeleton for uscogdata v0.1. Internal session layer with lazy
cog_open(), manifest fetch+validate+cache, view registration placeholder.
Depends on DuckDB >=1.0, httr2, jsonlite. inst/schemas/provenance-v1.json
ships the structured provenance JSON Schema.
---
 .Rbuildignore                   | 11 +++++++++
 .gitignore                      | 11 +++++++++
 DESCRIPTION                     | 36 +++++++++++++++++++++++++++
 LICENSE                         |  2 ++
 NAMESPACE                       |  2 ++
 R/cache.R                       |  6 +++++
 R/config.R                      | 29 ++++++++++++++++++++++
 R/manifest.R                    | 40 ++++++++++++++++++++++++++++++
 R/session.R                     | 43 +++++++++++++++++++++++++++++++++
 R/views.R                       | 13 ++++++++++
 R/zzz.R                         |  9 +++++++
 README.md                       | 26 ++++++++++++++++++++
 _pkgdown.yml                    | 13 ++++++++++
 inst/schemas/provenance-v1.json | 21 ++++++++++++++++
 inst/sql/.gitkeep               |  0
 tests/testthat.R                |  4 +++
 tests/testthat/helper-fixture.R |  6 +++++
 tests/testthat/setup.R          |  5 ++++
 tests/testthat/test-config.R    | 28 +++++++++++++++++++++
 vignettes/.gitkeep              |  0
 20 files changed, 305 insertions(+)
 create mode 100644 .Rbuildignore
 create mode 100644 .gitignore
 create mode 100644 DESCRIPTION
 create mode 100644 LICENSE
 create mode 100644 NAMESPACE
 create mode 100644 R/cache.R
 create mode 100644 R/config.R
 create mode 100644 R/manifest.R
 create mode 100644 R/session.R
 create mode 100644 R/views.R
 create mode 100644 R/zzz.R
 create mode 100644 README.md
 create mode 100644 _pkgdown.yml
 create mode 100644 inst/schemas/provenance-v1.json
 create mode 100644 inst/sql/.gitkeep
 create mode 100644 tests/testthat.R
 create mode 100644 tests/testthat/helper-fixture.R
 create mode 100644 tests/testthat/setup.R
 create mode 100644 tests/testthat/test-config.R
 create mode 100644 vignettes/.gitkeep

diff --git a/.Rbuildignore b/.Rbuildignore
new file mode 100644
index 0000000..46c7c80
--- /dev/null
+++ b/.Rbuildignore
@@ -0,0 +1,11 @@
+^.*\.Rproj$
+^\.Rproj\.user$
+^_pkgdown\.yml$
+^docs$
+^pkgdown$
+^\.github$
+^LICENSE\.md$
+^\.git$
+^\.gitignore$
+\.gitkeep$
+^vignettes$
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7beed34
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+*.Rproj
+inst/doc
+docs/
+/doc/
+/Meta/
+.DS_Store
+/.quarto/
diff --git a/DESCRIPTION b/DESCRIPTION
new file mode 100644
index 0000000..9c9566d
--- /dev/null
+++ b/DESCRIPTION
@@ -0,0 +1,36 @@
+Package: uscogdata
+Type: Package
+Title: Curated Reader for the Civilytics US Census of Governments Finance Corpus
+Version: 0.1.0
+Authors@R:
+    person("Civilytics", , , "jknowles@gmail.com", role = c("aut", "cre"))
+Description: Curated R verbs over the Civilytics US Census of Governments
+    finance corpus. Provides unit-level financial profiles, geographic
+    rollups, and peer comparisons with auditable provenance and built-in
+    cross-vintage correctness.
+License: MIT + file LICENSE
+Encoding: UTF-8
+LazyData: false
+Depends: R (>= 4.1)
+Imports:
+    DBI (>= 1.1.0),
+    dbplyr (>= 2.4.0),
+    duckdb (>= 1.0.0),
+    dplyr (>= 1.1.0),
+    tibble,
+    cli,
+    jsonlite,
+    httr2,
+    digest
+Suggests:
+    testthat (>= 3.0.0),
+    withr,
+    knitr,
+    rmarkdown,
+    pkgdown,
+    ggplot2
+Config/testthat/edition: 3
+VignetteBuilder: knitr
+RoxygenNote: 7.3.3
+MinCorpusSchema: 2
+MaxCorpusSchema: 2
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..3bb8b6e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,2 @@
+YEAR: 2026
+COPYRIGHT HOLDER: Civilytics
diff --git a/NAMESPACE b/NAMESPACE
new file mode 100644
index 0000000..6ae9268
--- /dev/null
+++ b/NAMESPACE
@@ -0,0 +1,2 @@
+# Generated by roxygen2: do not edit by hand
+
diff --git a/R/cache.R b/R/cache.R
new file mode 100644
index 0000000..cd3a3e8
--- /dev/null
+++ b/R/cache.R
@@ -0,0 +1,6 @@
+# R/cache.R
+# Local partition cache; SHA-based invalidation.
+# Phase N v0.1 implementation: DuckDB httpfs handles actual reads directly
+# from Nextcloud; cache_dir holds only manifest.json. Richer partition
+# caching (pre-fetch hot partitions) is a v0.2 feature.
+# Stub here for cog_mirror to compose against.
diff --git a/R/config.R b/R/config.R
new file mode 100644
index 0000000..6ebc78f
--- /dev/null
+++ b/R/config.R
@@ -0,0 +1,29 @@
+# R/config.R
+
+#' Package-private mutable state
+#' @noRd
+.uscogdata_env <- new.env(parent = emptyenv())
+
+.uscogdata_defaults <- list(
+  url               = "https://cloud.civilytics.org/s/REPLACE_WITH_SHARE_TOKEN/download/",
+  cache_dir         = NULL,
+  manifest_ttl_secs = 3600L
+)
+
+#' Resolve a config value: env var > option > default
+#' @noRd
+.cfg <- function(key) {
+  env_var <- paste0("USCOGDATA_", toupper(key))
+  v <- Sys.getenv(env_var, unset = NA)
+  if (!is.na(v) && nzchar(v)) return(v)
+  opt <- getOption(paste0("uscogdata.", key), default = NULL)
+  if (!is.null(opt)) return(opt)
+  .uscogdata_defaults[[key]]
+}
+
+.resolve_url <- function() .cfg("url")
+
+.resolve_cache_dir <- function() {
+  v <- .cfg("cache_dir")
+  if (is.null(v)) tools::R_user_dir("uscogdata", "cache") else v
+}
diff --git a/R/manifest.R b/R/manifest.R
new file mode 100644
index 0000000..9fa7c0d
--- /dev/null
+++ b/R/manifest.R
@@ -0,0 +1,40 @@
+# R/manifest.R
+
+#' Fetch manifest.json from URL, cache locally, validate TTL.
+#' @noRd
+.fetch_or_cache_manifest <- function(url, cache_dir) {
+  cache_path <- file.path(cache_dir, "manifest.json")
+  ttl <- as.integer(.cfg("manifest_ttl_secs"))
+
+  needs_fetch <- !file.exists(cache_path) ||
+                 difftime(Sys.time(), file.info(cache_path)$mtime, units = "secs") > ttl
+
+  if (needs_fetch) {
+    resp <- httr2::request(paste0(url, "manifest.json")) |>
+      httr2::req_error(is_error = function(r) httr2::resp_status(r) >= 400) |>
+      httr2::req_perform()
+    writeLines(httr2::resp_body_string(resp), cache_path)
+  }
+
+  jsonlite::fromJSON(cache_path, simplifyVector = FALSE)
+}
+
+#' @noRd
+.validate_schema <- function(manifest, expected_version) {
+  if (manifest$schema_version != expected_version) {
+    cli::cli_abort(c(
+      "Corpus schema version mismatch.",
+      x = "Package expects schema_version = {expected_version}; corpus has {manifest$schema_version}.",
+      i = "Update uscogdata (install.packages or pak::pkg_install) or re-publish corpus."
+    ))
+  }
+}
+
+#' @noRd
+.validate_scope <- function(manifest) {
+  included <- manifest$scope$gov_types_included
+  .uscogdata_env$scope_included <- included
+  invisible(NULL)
+}
+
+`%||%` <- function(a, b) if (is.null(a) || (length(a) == 1 && is.na(a))) b else a
diff --git a/R/session.R b/R/session.R
new file mode 100644
index 0000000..0d8147d
--- /dev/null
+++ b/R/session.R
@@ -0,0 +1,43 @@
+# R/session.R
+
+#' Internal: open session, register views, cache manifest.
+#' Not exported. Called lazily by verbs via .ensure_session().
+#' @noRd
+cog_open <- function(url = .resolve_url(),
+                     cache_dir = .resolve_cache_dir()) {
+  if (!dir.exists(cache_dir)) dir.create(cache_dir, recursive = TRUE)
+
+  con <- DBI::dbConnect(duckdb::duckdb())
+  DBI::dbExecute(con, "INSTALL httpfs; LOAD httpfs;")
+
+  manifest <- .fetch_or_cache_manifest(url, cache_dir)
+  .validate_schema(manifest, expected_version = 2L)
+  .validate_scope(manifest)
+
+  .register_views(con, url, manifest)
+
+  .uscogdata_env$con       <- con
+  .uscogdata_env$manifest  <- manifest
+  .uscogdata_env$url       <- url
+  .uscogdata_env$cache_dir <- cache_dir
+
+  invisible(con)
+}
+
+#' @noRd
+.ensure_session <- function() {
+  if (is.null(.uscogdata_env$con) ||
+      !DBI::dbIsValid(.uscogdata_env$con)) {
+    cog_open()
+  }
+  .uscogdata_env$con
+}
+
+#' @noRd
+cog_close <- function() {
+  if (!is.null(.uscogdata_env$con) && DBI::dbIsValid(.uscogdata_env$con)) {
+    DBI::dbDisconnect(.uscogdata_env$con, shutdown = TRUE)
+  }
+  .uscogdata_env$con      <- NULL
+  .uscogdata_env$manifest <- NULL
+}
diff --git a/R/views.R b/R/views.R
new file mode 100644
index 0000000..adfae8d
--- /dev/null
+++ b/R/views.R
@@ -0,0 +1,13 @@
+# R/views.R
+
+#' Register DuckDB views from inst/sql/ SQL files
+#' @noRd
+.register_views <- function(con, url, manifest) {
+  sql_dir <- system.file("sql", package = "uscogdata")
+  files   <- list.files(sql_dir, pattern = "\\.sql$", full.names = TRUE)
+  for (f in files) {
+    sql <- paste(readLines(f, warn = FALSE), collapse = "\n")
+    sql <- gsub("\\{url\\}", url, sql, fixed = FALSE)
+    DBI::dbExecute(con, sql)
+  }
+}
diff --git a/R/zzz.R b/R/zzz.R
new file mode 100644
index 0000000..8c4464c
--- /dev/null
+++ b/R/zzz.R
@@ -0,0 +1,9 @@
+# R/zzz.R
+
+.onLoad <- function(libname, pkgname) {
+  invisible(NULL)
+}
+
+.onUnload <- function(libpath) {
+  cog_close()
+}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..7275ed4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,26 @@
+# uscogdata
+
+Curated R reader for the Civilytics US Census of Governments finance corpus.
+
+Provides unit-level financial profiles, geographic rollups, and peer comparisons
+with auditable provenance and built-in cross-vintage correctness. Reads the
+published corpus (Hive-partitioned parquet + manifest.json) directly from
+Nextcloud via DuckDB httpfs — no local bulk downloads required.
+
+## Status
+
+Under active development (Phase 2 of the cog_pipeline project). See
+`../cog_pipeline/docs/reader-specification.md` for the reader contract this
+package implements.
+
+## Installation
+
+```r
+# pak::pkg_install("gitea.civilytics.org/Civilytics/uscogdata")
+```
+
+## Configuration
+
+- `USCOGDATA_URL` — corpus root URL (public Nextcloud share, trailing slash)
+- `USCOGDATA_CACHE_DIR` — optional override for the manifest cache directory
+- `USCOGDATA_MANIFEST_TTL_SECS` — optional manifest re-fetch TTL (default 3600)
diff --git a/_pkgdown.yml b/_pkgdown.yml
new file mode 100644
index 0000000..3aeefe7
--- /dev/null
+++ b/_pkgdown.yml
@@ -0,0 +1,13 @@
+url: ~
+template:
+  bootstrap: 5
+
+reference:
+  - title: Session
+    contents:
+      - has_keyword("internal")
+
+articles:
+  - title: Getting started
+    navbar: ~
+    contents: []
diff --git a/inst/schemas/provenance-v1.json b/inst/schemas/provenance-v1.json
new file mode 100644
index 0000000..e970a3f
--- /dev/null
+++ b/inst/schemas/provenance-v1.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://civilytics.org/schemas/uscogdata/provenance-v1.json",
+  "title": "uscogdata provenance v1",
+  "type": "object",
+  "required": ["verb", "target", "years", "scope", "manifest", "sql_query"],
+  "properties": {
+    "verb":           { "type": "string" },
+    "call":           { "type": "string" },
+    "target":         { "type": "object" },
+    "years":          { "type": "array", "items": { "type": "integer" } },
+    "category":       { "type": ["string", "array", "null"] },
+    "scope":          { "type": "object" },
+    "codes_summed":   { "type": "object" },
+    "aggregate_fallback": { "type": ["object", "null"] },
+    "transformations":{ "type": "object" },
+    "series_break_refs": { "type": "array", "items": { "type": "string" } },
+    "manifest":       { "type": "object" },
+    "sql_query":      { "type": "string" }
+  }
+}
diff --git a/inst/sql/.gitkeep b/inst/sql/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/tests/testthat.R b/tests/testthat.R
new file mode 100644
index 0000000..c579ba2
--- /dev/null
+++ b/tests/testthat.R
@@ -0,0 +1,4 @@
+library(testthat)
+library(uscogdata)
+
+test_check("uscogdata")
diff --git a/tests/testthat/helper-fixture.R b/tests/testthat/helper-fixture.R
new file mode 100644
index 0000000..f63d983
--- /dev/null
+++ b/tests/testthat/helper-fixture.R
@@ -0,0 +1,6 @@
+# tests/testthat/helper-fixture.R
+skip_if_no_corpus <- function() {
+  testthat::skip_if(Sys.getenv("USCOGDATA_FIXTURE_URL", "") == "" &&
+                    !file.exists("~/.cache/R/uscogdata/manifest.json"),
+                    "No fixture corpus available")
+}
diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R
new file mode 100644
index 0000000..b774389
--- /dev/null
+++ b/tests/testthat/setup.R
@@ -0,0 +1,5 @@
+# tests/testthat/setup.R
+# Point tests at a fixture corpus URL if provided.
+if (Sys.getenv("USCOGDATA_FIXTURE_URL", "") != "") {
+  options(uscogdata.url = Sys.getenv("USCOGDATA_FIXTURE_URL"))
+}
diff --git a/tests/testthat/test-config.R b/tests/testthat/test-config.R
new file mode 100644
index 0000000..8356046
--- /dev/null
+++ b/tests/testthat/test-config.R
@@ -0,0 +1,28 @@
+test_that(".cfg resolves defaults, options, and env vars in priority order", {
+  withr::with_options(list(uscogdata.manifest_ttl_secs = NULL), {
+    withr::with_envvar(c(USCOGDATA_MANIFEST_TTL_SECS = NA), {
+      expect_equal(uscogdata:::.cfg("manifest_ttl_secs"), 3600L)
+    })
+  })
+
+  withr::with_options(list(uscogdata.url = "https://opt.example/"), {
+    withr::with_envvar(c(USCOGDATA_URL = NA), {
+      expect_equal(uscogdata:::.cfg("url"), "https://opt.example/")
+    })
+  })
+
+  withr::with_envvar(c(USCOGDATA_URL = "https://env.example/"), {
+    withr::with_options(list(uscogdata.url = "https://opt.example/"), {
+      expect_equal(uscogdata:::.cfg("url"), "https://env.example/")
+    })
+  })
+})
+
+test_that(".resolve_cache_dir falls back to R_user_dir", {
+  withr::with_envvar(c(USCOGDATA_CACHE_DIR = NA), {
+    withr::with_options(list(uscogdata.cache_dir = NULL), {
+      expect_equal(uscogdata:::.resolve_cache_dir(),
+                   tools::R_user_dir("uscogdata", "cache"))
+    })
+  })
+})
diff --git a/vignettes/.gitkeep b/vignettes/.gitkeep
new file mode 100644
index 0000000..e69de29