- Rate per 1,000 students labeled above each observed point. The modeled total sums each
- selected group's median/interval independently, which approximates but is not exactly the
- median of the combined total. Data from CRDC waves 2015–16 through 2021–22.
+ Rate per 1,000 students labeled above each observed point. Data from CRDC waves 2015–16
+ through 2021–22.
)
diff --git a/src/components/ChartPanel.jsx b/src/components/ChartPanel.jsx
index 93c78f0..a5dc956 100644
--- a/src/components/ChartPanel.jsx
+++ b/src/components/ChartPanel.jsx
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useMemo, useState } from 'react'
import * as api from '../hooks/useApi.js'
import { MODEL_QUADRANTS } from '../hooks/useApi.js'
-import { useDrawDistribution } from '../hooks/useDrawDistribution.js'
+import { useDistrictTotalDraws, useDrawDistribution } from '../hooks/useDrawDistribution.js'
import {
buildDisplayGroups,
defaultDiffPair,
@@ -20,6 +20,11 @@ const DEFAULT_SPEC = 'unified_m4_mod' // three-year + referral rate
const CURRENT_WAVE = '21-22'
const QUADRANT_MODELS = MODEL_QUADRANTS.map((q) => q.model)
+// Below this, the draws for all three waves are fetched without asking. Sized
+// from the real shards: Nevada's three waves are 0.31MB, California's 13.9MB
+// and Texas's 14.3MB, so 3MB cleanly separates "free" from "worth a click".
+const AUTO_TOTAL_DRAW_BYTES = 3 * 1024 * 1024
+
/**
* The results page: what was observed (summary table), what the model says the
* rate is (density panel), and what it says about the gap between two groups
@@ -43,6 +48,7 @@ export default function ChartPanel({ district, state }) {
// made in one mode must not be reapplied in the other.
const [selection, setSelection] = useState(null)
const [pairSelection, setPairSelection] = useState(null)
+ const [exactTotals, setExactTotals] = useState(false)
// ——— Fetch: the three waves for the time series ———
useEffect(() => {
@@ -54,6 +60,7 @@ export default function ChartPanel({ district, state }) {
setSelection(null)
setPairSelection(null)
setCompareAll(false)
+ setExactTotals(false)
async function run() {
const waves = {}
@@ -154,6 +161,22 @@ export default function ChartPanel({ district, state }) {
year: CURRENT_WAVE,
})
+ // The time series' band, computed from the draws rather than by summing each
+ // group's own bounds. Fetched automatically only where it is cheap — three
+ // waves is 0.3MB for Nevada but 13.9MB for California — otherwise the chart
+ // offers it as an explicit choice with the size shown.
+ const totals = useDistrictTotalDraws({
+ leaid: district.leaid,
+ state,
+ model: WAVE_MODEL,
+ years: ALL_WAVES,
+ enabled: exactTotals,
+ })
+
+ useEffect(() => {
+ if (totals.bytes > 0 && totals.bytes <= AUTO_TOTAL_DRAW_BYTES) setExactTotals(true)
+ }, [totals.bytes])
+
if (loading || !waveData) return setExactTotals(true)}
+ />
>} parquet parts, in order
+ */
+function listShardEntries(model, year, state) {
+ const dir = shardDir(model, year, state)
+ if (!listingCache.has(dir)) {
+ listingCache.set(
+ dir,
+ (async () => {
+ try {
+ const res = await fetch(`${HF_TREE}/${dir}`)
+ if (!res.ok) throw new Error(`tree listing HTTP ${res.status}`)
+ const entries = await res.json()
+ return entries
+ .filter((e) => e?.type === 'file' && /\/data_\d+\.parquet$/.test(e.path || ''))
+ .map((e) => ({ part: Number(e.path.match(/data_(\d+)\.parquet$/)[1]), size: e.size || 0 }))
+ .sort((a, b) => a.part - b.part)
+ } catch (err) {
+ listingCache.delete(dir)
+ throw err
+ }
+ })(),
+ )
+ }
+ return listingCache.get(dir)
+}
+
+/**
+ * Total bytes of the draw shards for one model across several years — what a
+ * "compute this from the draws" action will actually download.
+ *
+ * @returns {Promise} bytes, or 0 if the size can't be determined
+ */
+export async function measureShardBytes(model, years, state) {
+ try {
+ const sizes = await Promise.all(
+ years.map(async (year) =>
+ (await listShardEntries(model, year, state)).reduce((sum, p) => sum + p.size, 0),
+ ),
+ )
+ return sizes.reduce((sum, n) => sum + n, 0)
+ } catch {
+ return 0
+ }
+}
+
/**
* Lists the parquet parts published for one (model, year, state).
*
@@ -42,13 +94,7 @@ function shardDir(model, year, state) {
async function listShardParts(model, year, state) {
const dir = shardDir(model, year, state)
try {
- const res = await fetch(`${HF_TREE}/${dir}`)
- if (!res.ok) throw new Error(`tree listing HTTP ${res.status}`)
- const entries = await res.json()
- const parts = entries
- .filter((e) => e?.type === 'file' && /\/data_\d+\.parquet$/.test(e.path || ''))
- .map((e) => ({ part: Number(e.path.match(/data_(\d+)\.parquet$/)[1]), path: e.path }))
- .sort((a, b) => a.part - b.part)
+ const parts = await listShardEntries(model, year, state)
if (parts.length > 0) return parts.map((p) => p.part)
throw new Error('tree listing contained no parquet parts')
} catch (err) {
@@ -139,12 +185,18 @@ async function fetchModelCounts(db, { leaid, state, model, year }) {
// a short draw set: its density and interval would be computed off a
// biased subsample and look identical on screen to a complete one.
const counts = {}
+ let dropped = 0
for (const [key, arr] of Object.entries(raw)) {
if (isCompleteDrawSet(arr, nDraws)) counts[key] = arr
- else console.warn('useDrawDistribution: dropping incomplete draw set for', { leaid, model, key, got: arr.length, want: nDraws })
+ else {
+ dropped += 1
+ console.warn('useDrawDistribution: dropping incomplete draw set for', { leaid, model, key, got: arr.length, want: nDraws })
+ }
}
- return Object.keys(counts).length > 0 ? { counts, nDraws } : null
+ // `dropped` matters to any consumer that aggregates *across* groups (the
+ // district total): a missing group there is an undercount, not a gap.
+ return Object.keys(counts).length > 0 ? { counts, nDraws, dropped } : null
} finally {
if (conn) await conn.close()
}
@@ -257,3 +309,89 @@ export function useDrawDistribution({ leaid, state, models, year }) {
return { status, byModel, nDraws }
}
+
+/**
+ * The district's total arrest count per wave, with a 95% interval computed from
+ * the draws — summed within each draw, then summarized across draws.
+ *
+ * Gated behind `enabled` because the cost is wildly uneven: three waves of
+ * Nevada is 0.3MB, but California is 13.9MB and Texas 14.3MB. `bytes` is
+ * probed from the directory listings up front (~1KB per year) so the caller can
+ * either fetch automatically when it's cheap or show the reader the price
+ * first.
+ *
+ * @param {{leaid: string, state: string, model: string, years: string[], enabled: boolean}} params
+ * @returns {{status: 'idle'|'loading'|'ready'|'error',
+ * byYear: Record | null,
+ * bytes: number}}
+ */
+export function useDistrictTotalDraws({ leaid, state, model, years, enabled }) {
+ const [status, setStatus] = useState('idle')
+ const [byYear, setByYear] = useState(null)
+ const [bytes, setBytes] = useState(0)
+
+ const yearsSignature = (years || []).join(',')
+
+ // Probe sizes regardless of `enabled` — this is what lets the UI decide.
+ useEffect(() => {
+ let cancelled = false
+ setBytes(0)
+ if (!state || !model || !yearsSignature) return
+ measureShardBytes(model, yearsSignature.split(','), state).then((n) => {
+ if (!cancelled) setBytes(n)
+ })
+ return () => {
+ cancelled = true
+ }
+ }, [state, model, yearsSignature])
+
+ useEffect(() => {
+ let cancelled = false
+ setStatus(enabled ? 'loading' : 'idle')
+ setByYear(null)
+
+ if (!enabled || !leaid || !state || !model || !yearsSignature) return
+
+ async function run() {
+ try {
+ const db = await getDb()
+ const yearList = yearsSignature.split(',')
+ const results = await Promise.all(
+ yearList.map(async (year) => {
+ try {
+ const model_ = await fetchModelCounts(db, { leaid, state, model, year })
+ if (!model_ || model_.dropped > 0) return [year, null]
+ const interval = totalInterval(totalPerDraw(model_.counts, model_.nDraws))
+ return [year, interval]
+ } catch (err) {
+ console.error(`useDistrictTotalDraws: ${year} failed:`, err)
+ return [year, null]
+ }
+ }),
+ )
+ if (cancelled) return
+ const next = Object.fromEntries(results)
+ if (Object.values(next).some((v) => v !== null)) {
+ setByYear(next)
+ setStatus('ready')
+ } else {
+ setByYear(null)
+ setStatus('error')
+ }
+ } catch (err) {
+ console.error('useDistrictTotalDraws failed:', err)
+ if (!cancelled) {
+ setByYear(null)
+ setStatus('error')
+ }
+ }
+ }
+
+ run()
+ return () => {
+ cancelled = true
+ }
+ }, [leaid, state, model, yearsSignature, enabled])
+
+ return { status, byYear, bytes }
+}
diff --git a/src/utils/districtTotal.js b/src/utils/districtTotal.js
new file mode 100644
index 0000000..62eacf9
--- /dev/null
+++ b/src/utils/districtTotal.js
@@ -0,0 +1,70 @@
+/**
+ * The district-wide arrest total, taken from the posterior predictive draws.
+ *
+ * Why this exists: the time-series chart used to draw its band by summing each
+ * student group's own `count_lower`/`count_upper`. That is not the interval of
+ * the total. The sum of per-group 97.5th percentiles is the value you would see
+ * if *every* group simultaneously landed at its own extreme in the same draw,
+ * which is far less likely than any one group doing so — so the band came out
+ * systematically too wide. Summing within each draw and taking quantiles of the
+ * resulting totals answers the actual question: how many arrests does the model
+ * think this district had?
+ *
+ * This is `build_state_summary()`'s method from crdc-arrests/R/summarize_draws.R
+ * applied on a different axis — sum inside the draw, summarize across draws —
+ * the same operation `poolBySex` performs for sex pooling.
+ *
+ * One caveat to carry into any caption. Upstream, `draw_id` is renumbered per
+ * write batch, so draw *k* of one group is not the same posterior sample as
+ * draw *k* of another; measured cross-group correlation is ≈ 0.02. Summing at a
+ * draw index therefore convolves what are effectively independent draws. That
+ * is a reasonable model here — these are *posterior predictive* draws, whose
+ * observation noise is independent across groups by construction and dominates
+ * the parameter-level covariance — but it does mean the result should be read
+ * as a predictive total, not as a contrast that preserves parameter
+ * correlation. It is still much closer to the truth than summing bounds.
+ */
+
+import { quantile } from './kde.js'
+import { isCompleteDrawSet } from './drawGroups.js'
+
+/** Matches the API's default interval, so the two are directly comparable. */
+export const TOTAL_INTERVAL_MASS = 0.95
+
+/**
+ * District total at each draw index: the sum across every student group.
+ *
+ * Returns null unless *every* group has a complete draw set. A group silently
+ * missing from the sum would undercount the total at every draw and shift the
+ * whole interval down, which is indistinguishable on screen from a real result.
+ *
+ * @param {Record | null | undefined} countsByGroup
+ * @param {number} nDraws
+ * @returns {number[] | null}
+ */
+export function totalPerDraw(countsByGroup, nDraws) {
+ const groups = Object.values(countsByGroup || {})
+ if (!groups.length || !(nDraws > 0)) return null
+ if (!groups.every((counts) => isCompleteDrawSet(counts, nDraws))) return null
+
+ const totals = new Array(nDraws).fill(0)
+ for (const counts of groups) {
+ for (let i = 0; i < nDraws; i++) totals[i] += counts[i]
+ }
+ return totals
+}
+
+/**
+ * @param {number[] | null | undefined} totals - district total per draw
+ * @returns {{lower: number, median: number, upper: number, nDraws: number} | null}
+ */
+export function totalInterval(totals) {
+ if (!totals?.length) return null
+ const tail = (1 - TOTAL_INTERVAL_MASS) / 2
+ return {
+ lower: quantile(totals, tail),
+ median: quantile(totals, 0.5),
+ upper: quantile(totals, 1 - tail),
+ nDraws: totals.length,
+ }
+}
diff --git a/src/utils/districtTotal.test.js b/src/utils/districtTotal.test.js
new file mode 100644
index 0000000..a63370f
--- /dev/null
+++ b/src/utils/districtTotal.test.js
@@ -0,0 +1,86 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { TOTAL_INTERVAL_MASS, totalInterval, totalPerDraw } from './districtTotal.js'
+
+// ——— totalPerDraw ———
+
+test('totalPerDraw: sums every group at the same draw index', () => {
+ const counts = { BL_F: [1, 2, 3], BL_M: [10, 20, 30], WH_F: [100, 200, 300] }
+ assert.deepEqual(totalPerDraw(counts, 3), [111, 222, 333])
+})
+
+test('totalPerDraw: a single group is its own total', () => {
+ assert.deepEqual(totalPerDraw({ BL_M: [4, 5] }, 2), [4, 5])
+})
+
+test('totalPerDraw: null when any group is short of nDraws', () => {
+ // Summing a 2-draw group into a 3-draw total would silently undercount the
+ // last draw, biasing the whole interval downward.
+ assert.equal(totalPerDraw({ BL_F: [1, 2, 3], BL_M: [1, 2] }, 3), null)
+})
+
+test('totalPerDraw: null when a group has a hole', () => {
+ const holey = [1, 2, 3]
+ delete holey[1]
+ assert.equal(totalPerDraw({ BL_F: holey }, 3), null)
+})
+
+test('totalPerDraw: null for empty or missing input', () => {
+ assert.equal(totalPerDraw({}, 500), null)
+ assert.equal(totalPerDraw(null, 500), null)
+ assert.equal(totalPerDraw({ BL_F: [1] }, 0), null)
+})
+
+test('totalPerDraw: does not mutate its input', () => {
+ const counts = { BL_F: [1, 2], BL_M: [3, 4] }
+ totalPerDraw(counts, 2)
+ assert.deepEqual(counts, { BL_F: [1, 2], BL_M: [3, 4] })
+})
+
+// ——— totalInterval ———
+
+test('totalInterval: median and 95% bounds from the draw totals', () => {
+ const totals = Array.from({ length: 1001 }, (_, i) => i) // 0…1000
+ const iv = totalInterval(totals)
+ // Linear-interpolated quantiles, so compare with a tolerance rather than
+ // exactly: p=0.025 over 0…1000 lands on 25 ± float noise.
+ const near = (a, b) => Math.abs(a - b) < 1e-9
+ assert.ok(near(iv.median, 500), `median ${iv.median}`)
+ assert.ok(near(iv.lower, 25), `lower ${iv.lower}`)
+ assert.ok(near(iv.upper, 975), `upper ${iv.upper}`)
+ assert.equal(iv.nDraws, 1001)
+})
+
+test('totalInterval: uses a 95% mass, matching the API convention', () => {
+ assert.equal(TOTAL_INTERVAL_MASS, 0.95)
+})
+
+test('totalInterval: lower <= median <= upper', () => {
+ const totals = Array.from({ length: 500 }, (_, i) => (i * 7919) % 331)
+ const iv = totalInterval(totals)
+ assert.ok(iv.lower <= iv.median && iv.median <= iv.upper)
+})
+
+test('totalInterval: a degenerate total collapses to a point', () => {
+ const iv = totalInterval(new Array(500).fill(12))
+ assert.deepEqual([iv.lower, iv.median, iv.upper], [12, 12, 12])
+})
+
+test('totalInterval: null for empty or missing totals', () => {
+ assert.equal(totalInterval([]), null)
+ assert.equal(totalInterval(null), null)
+})
+
+test('totalInterval: is narrower than summing the groups own bounds', () => {
+ // The property that motivates this whole path. Two independent groups, each
+ // roughly uniform on 0..100: summing each group's 97.5th percentile gives
+ // ~200, but the 97.5th percentile of the *sum* is well below that, because
+ // both groups landing at their extreme in the same draw is rare.
+ const n = 4000
+ const a = Array.from({ length: n }, (_, i) => (i * 37) % 101)
+ const b = Array.from({ length: n }, (_, i) => (i * 61) % 101)
+ const iv = totalInterval(totalPerDraw({ A: a, B: b }, n))
+ const summedBounds = { lower: 0, upper: 100 + 100 }
+ assert.ok(iv.upper < summedBounds.upper, `sum-of-quantiles ${summedBounds.upper} vs quantile-of-sum ${iv.upper}`)
+ assert.ok(iv.upper - iv.lower < summedBounds.upper - summedBounds.lower)
+})