+ Δ is computed at each of {summary.n.toLocaleString()} posterior predictive draws as{' '}
+ {groupA.label} minus {groupB.label}, per 1,000 students. The dashed line marks no
+ difference. Bars beneath the curve are the 80% and 95% intervals around the median Δ.
+
+ In {formatPercent(share)} of posterior predictive draws, the {higher.sentenceLabel} arrest
+ rate exceeds the {lower.sentenceLabel} rate. The median difference is{' '}
+ {formatDelta(summary.median)} per 1,000 students.
+
- )
-}
diff --git a/src/charts/RateDensityPanel.jsx b/src/charts/RateDensityPanel.jsx
new file mode 100644
index 0000000..c45692b
--- /dev/null
+++ b/src/charts/RateDensityPanel.jsx
@@ -0,0 +1,532 @@
+import { useMemo } from 'react'
+import { area, curveMonotoneX, curveStep } from 'd3-shape'
+import { scaleLinear } from 'd3-scale'
+import { MODEL_QUADRANTS } from '../hooks/useApi.js'
+import { raceColor } from '../utils/colors.js'
+import { agrestiCoull } from '../utils/agrestiCoull.js'
+import { densityProfile } from '../utils/densityProfile.js'
+import { displayDraws } from '../utils/districtGroups.js'
+import { computeRateDomain } from '../utils/rateDomain.js'
+import { toRates } from '../utils/pooling.js'
+import { densityCurve, fitSkewedInterval } from '../utils/distributionApprox.js'
+import ApproxNote from '../components/ApproxNote.jsx'
+
+/**
+ * Chart A — "Arrest rate probability density".
+ *
+ * Port of the white paper's Fig 6 (`wp_fig_group_density` in
+ * crdc-arrests/R/paper_figures.R): each selected student group's posterior
+ * predictive arrest rate as a filled density, with that group's frequentist
+ * Agresti–Coull interval on a rail beneath, so model and observation can be
+ * read against each other.
+ *
+ * Layout follows the palette contract in utils/colors.js: **race is hue, sex is
+ * position**. Female and Male are two stacked sub-panels sharing one x-axis,
+ * never two hues; when sex pooling is on there is a single panel and the sex
+ * dimension disappears entirely rather than being recoloured.
+ *
+ * React owns the DOM here — d3 supplies scales and path generators only. No
+ * selections, no imperative mutation.
+ */
+
+const AGRESTI_COULL_LEVEL = 0.95
+const FILL_OPACITY = 0.4
+const STROKE_WIDTH = 2
+
+const CHART_WIDTH = 760
+const MARGIN = { top: 22, right: 18, bottom: 34, left: 16 }
+const ROW_HEIGHT = 118
+const COMPARE_ROW_HEIGHT = 74
+const RAIL_HEIGHT = 26
+
+export default function RateDensityPanel({
+ groups,
+ selectedKeys,
+ enrollByGroup,
+ byModel,
+ status,
+ nDraws,
+ selectedModel,
+ onSelectModel,
+ compareAll,
+ onCompareAllChange,
+ pooled,
+}) {
+ const activeModels = useMemo(
+ () => (compareAll ? MODEL_QUADRANTS.map((q) => q.model) : [selectedModel]),
+ [compareAll, selectedModel],
+ )
+
+ const selected = useMemo(
+ () => groups.filter((g) => selectedKeys.includes(g.key)),
+ [groups, selectedKeys],
+ )
+
+ // Agresti–Coull is computed from observed counts, so it is identical across
+ // model specifications — the same rail is drawn on every model row, which is
+ // exactly what makes the rows comparable.
+ const acByKey = useMemo(() => {
+ const out = {}
+ for (const g of selected) {
+ if (!(g.enroll > 0)) continue
+ const ac = agrestiCoull(g.observed, g.enroll, AGRESTI_COULL_LEVEL)
+ out[g.key] = {
+ // The R original can return a negative lower bound for a single event;
+ // a negative arrest count is not drawable, so clamp here rather than in
+ // the port (see utils/agrestiCoull.js).
+ lower: Math.max(0, (ac.lower / g.enroll) * 1000),
+ upper: (ac.upper / g.enroll) * 1000,
+ point: (g.observed / g.enroll) * 1000,
+ }
+ }
+ return out
+ }, [selected])
+
+ // Draws for every active model, moved into display (pooled or unpooled) space.
+ const drawsByModel = useMemo(() => {
+ const out = {}
+ for (const model of activeModels) {
+ out[model] = displayDraws(byModel?.[model]?.counts, enrollByGroup, pooled)
+ }
+ return out
+ }, [activeModels, byModel, enrollByGroup, pooled])
+
+ const domain = useMemo(() => {
+ const rateArrays = []
+ for (const model of activeModels) {
+ const { counts, enroll } = drawsByModel[model] || {}
+ for (const g of selected) rateArrays.push(toRates(counts?.[g.key], enroll?.[g.key]))
+ }
+ return computeRateDomain(rateArrays, Object.values(acByKey).map((a) => a.upper))
+ }, [activeModels, drawsByModel, selected, acByKey])
+
+ // One entry per (model, group): the shape to draw and where it came from.
+ const rowsByModel = useMemo(() => {
+ const out = {}
+ for (const model of activeModels) {
+ const { counts, enroll } = drawsByModel[model] || {}
+ out[model] = selected.map((g) => ({
+ group: g,
+ ...profileFor(g, counts?.[g.key], enroll?.[g.key], domain.niceMax),
+ }))
+ }
+ return out
+ }, [activeModels, drawsByModel, selected, domain.niceMax])
+
+ const anyApproximated = Object.values(rowsByModel)
+ .flat()
+ .some((r) => r.source !== 'draws')
+ const loading = status === 'loading'
+
+ const sexPanels = pooled
+ ? [{ sex: null, label: null }]
+ : [{ sex: 'F', label: 'Female students' }, { sex: 'M', label: 'Male students' }]
+
+ return (
+
+
+
+
Arrest rate probability density
+
+ Each curve is one student group’s modelled arrest rate. The bar beneath each panel
+ is what was actually reported, with its {Math.round(AGRESTI_COULL_LEVEL * 100)}%
+ Agresti–Coull interval.
+
+ The axis stops at {domain.niceMax} per 1,000; one or more groups extend beyond it. Those
+ curves and intervals are cut off at the right edge, marked ›.
+
+ )}
+
+
+ Densities are drawn from {nDraws > 0 ? nDraws.toLocaleString() : '500'} posterior predictive
+ draws per group. Point ranges are the observed rate with its{' '}
+ {Math.round(AGRESTI_COULL_LEVEL * 100)}% Agresti–Coull interval. Source: CRDC School
+ Arrest Rate API (Knowles & Miller 2025), 2021–22 Civil Rights Data Collection.
+
+
+ )
+}
+
+/**
+ * Picks the shape for one group: real draws when we have them, the fitted
+ * summary-interval approximation when we don't, and nothing at all for a pooled
+ * group with no draws — there is no honest pooled shape to fit (see
+ * `buildDisplayGroups`).
+ */
+function profileFor(group, counts, enroll, niceMax) {
+ const domain = { min: 0, max: niceMax, n: 60 }
+ const fromDraws = densityProfile(counts, enroll, domain)
+ if (fromDraws) return { profile: fromDraws, source: 'draws' }
+
+ if (!group.modeled || !(group.modeled.upper > 0)) return { profile: null, source: 'none' }
+
+ const fit = fitSkewedInterval({ ...group.modeled, intervalMass: 0.95 })
+ const points = densityCurve(fit, domain)
+ return {
+ profile: { kind: 'kde', points, maxY: Math.max(...points.map((p) => p.y)), step: 0 },
+ source: 'approx',
+ }
+}
+
+function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, compareAll }) {
+ const rows = activeModels.map((model) => ({
+ model,
+ label: MODEL_QUADRANTS.find((q) => q.model === model)?.label || model,
+ entries: (rowsByModel[model] || []).filter((r) => (sex ? r.group.sex === sex : true)),
+ }))
+
+ const hasAnything = rows.some((r) => r.entries.some((e) => e.profile))
+ const rowHeight = compareAll ? COMPARE_ROW_HEIGHT : ROW_HEIGHT
+ const innerWidth = CHART_WIDTH - MARGIN.left - MARGIN.right
+ const bandHeight = rowHeight + RAIL_HEIGHT
+ const height = MARGIN.top + rows.length * bandHeight + MARGIN.bottom
+
+ const x = scaleLinear().domain([0, domain.niceMax]).range([MARGIN.left, MARGIN.left + innerWidth])
+
+ return (
+
+ {label && {label}}
+ {!hasAnything ? (
+
No modelled distribution is available for these groups.
+ ) : (
+
+
+
+ )}
+
+ )
+}
+
+function ModelRow({ row, compareAll, x, top, rowHeight, acByKey, niceMax }) {
+ const baselineY = top + rowHeight
+ const peakHeight = rowHeight * 0.86
+ const drawable = row.entries.filter((e) => e.profile)
+
+ return (
+
+ {compareAll && (
+
+ {row.label}
+
+ )}
+
+ {drawable.map((entry, i) => (
+
+ ))}
+
+
+
+ {row.entries.map((entry, i) => {
+ const ac = acByKey[entry.group.key]
+ if (!ac) return null
+ return (
+
+ )
+ })}
+
+ )
+}
+
+function GroupArea({ entry, x, baselineY, peakHeight, labelIndex, compact, niceMax }) {
+ const { profile, group } = entry
+ const color = raceColor(group.race)
+ const y = scaleLinear().domain([0, profile.maxY || 1]).range([baselineY, baselineY - peakHeight])
+
+ const points =
+ profile.kind === 'mass'
+ ? padMassPoints(profile.points, profile.step, niceMax)
+ : profile.points
+
+ const areaGen = area()
+ .x((p) => x(Math.min(p.x, niceMax)))
+ .y0(baselineY)
+ .y1((p) => y(p.y))
+ // A staircase for discrete mass, a smooth curve for a KDE. curveStep keeps
+ // the mass profile honest — it says "these values and no others" — while
+ // still visually rhyming with the filled areas beside it.
+ .curve(profile.kind === 'mass' ? curveStep : curveMonotoneX)
+
+ const peak = profile.points.reduce((best, p) => (p.y > best.y ? p : best), profile.points[0])
+ const peakX = x(Math.min(peak.x, niceMax))
+ const labelY = Math.max(baselineY - peakHeight - 2, y(peak.y) - 4 - (labelIndex % 2) * 11)
+
+ return (
+
+
+
+ {/* Direct-labelled at the peak so there is no legend to hunt through. */}
+ x(niceMax) * 0.8 ? 'end' : 'middle'}
+ fontSize={compact ? '0.58rem' : '0.65rem'}
+ fontWeight={700}
+ fill={color}
+ stroke="var(--cv-paper)"
+ strokeWidth={3}
+ paintOrder="stroke"
+ >
+ {group.shortLabel}
+
+
+ )
+}
+
+/**
+ * A staircase needs a floor to close against on both sides: without the zero
+ * pads, curveStep leaves the first and last bars open and the fill bleeds to
+ * the panel edge.
+ */
+function padMassPoints(points, step, niceMax) {
+ const half = Math.max(step, 1e-6) / 2
+ const first = points[0]
+ const last = points[points.length - 1]
+ return [
+ { x: Math.max(0, first.x - half), y: 0 },
+ ...points,
+ { x: Math.min(last.x + half, niceMax), y: 0 },
+ ]
+}
+
+function PointRange({ ac, color, x, y, niceMax }) {
+ const lo = Math.min(ac.lower, niceMax)
+ const hi = Math.min(ac.upper, niceMax)
+ const point = Math.min(ac.point, niceMax)
+ const clipped = ac.upper > niceMax
+ return (
+
+
+
+ {clipped && (
+
+ ›
+
+ )}
+
+ )
+}
+
+function SpecControls({ selectedModel, onSelectModel, compareAll, onCompareAllChange, loading }) {
+ return (
+
- )
-}
diff --git a/src/components/ChartPanel.jsx b/src/components/ChartPanel.jsx
index 0b12deb..4f6fa8f 100644
--- a/src/components/ChartPanel.jsx
+++ b/src/components/ChartPanel.jsx
@@ -1,58 +1,163 @@
-import { useState, useEffect } from 'react'
+import { useCallback, useEffect, useMemo, useState } from 'react'
import * as api from '../hooks/useApi.js'
-import { groupLabel } from '../utils/colors.js'
+import { MODEL_QUADRANTS } from '../hooks/useApi.js'
+import { useDrawDistribution } from '../hooks/useDrawDistribution.js'
+import {
+ buildDisplayGroups,
+ defaultDiffPair,
+ defaultSelectedKeys,
+ enrollByGroupKey,
+} from '../utils/districtGroups.js'
+import { shouldPoolBySex } from '../utils/pooling.js'
import ArrestsOverTime from '../charts/ArrestsOverTime.jsx'
-import RateByGroupBar from '../charts/RateByGroupBar.jsx'
-import RateDensityRidgeline from '../charts/RateDensityRidgeline.jsx'
+import GroupDifference from '../charts/GroupDifference.jsx'
+import RateDensityPanel from '../charts/RateDensityPanel.jsx'
+import DistrictSummaryTable from './DistrictSummaryTable.jsx'
const ALL_WAVES = ['15-16', '17-18', '21-22']
-const QUADRANT_MODELS = ['unified_m1_mod', 'unified_m2_mod', 'unified_m3_mod', 'unified_m4_mod']
-const WAVE_MODEL = 'unified_m3_mod' // three-year, no covariate — powers Charts 1 & 2
+const WAVE_MODEL = 'unified_m3_mod' // three-year, no covariate — powers the time series
+const DEFAULT_SPEC = 'unified_m4_mod' // three-year + referral rate
+const CURRENT_WAVE = '21-22'
+const QUADRANT_MODELS = MODEL_QUADRANTS.map((q) => q.model)
/**
- * ChartPanel — 3 charts: arrests over time (observed vs. modeled), arrest
- * rate by student group (Female/Male panels), and the model-selectable
- * posterior density ridge chart.
+ * The results page: what was observed (summary table), what the model says the
+ * rate is (density panel), and what it says about the gap between two groups
+ * (difference chart) — plus the district's arrest history.
+ *
+ * This component owns every piece of cross-chart state: the selected model
+ * specification, whether sexes are pooled, which groups are checked, and which
+ * pair is being differenced. The charts are given values and callbacks and hold
+ * none of it, so the table's checkboxes and the density panel can't drift apart.
*/
export default function ChartPanel({ district, state }) {
- const [data, setData] = useState(null) // all fetched estimate rows keyed by year/model
+ const [waveData, setWaveData] = useState(null)
+ const [summaryByModel, setSummaryByModel] = useState({})
const [loading, setLoading] = useState(true)
+ const [selectedModel, setSelectedModel] = useState(DEFAULT_SPEC)
+ const [compareAll, setCompareAll] = useState(false)
+ const [poolOverride, setPoolOverride] = useState(null) // null = follow the rule
+ // Selections are stored with the pooling mode they were made in: pooled keys
+ // ('BL') and unpooled keys ('BL_F') are different namespaces, so a selection
+ // made in one mode must not be reapplied in the other.
+ const [selection, setSelection] = useState(null)
+ const [pairSelection, setPairSelection] = useState(null)
+
+ // ——— Fetch: the three waves for the time series ———
useEffect(() => {
- async function fetchData() {
+ let cancelled = false
+ setLoading(true)
+ setWaveData(null)
+ setSummaryByModel({})
+ setPoolOverride(null)
+ setSelection(null)
+ setPairSelection(null)
+ setCompareAll(false)
+
+ async function run() {
+ const waves = {}
+ await Promise.all(
+ ALL_WAVES.map(async (year) => {
+ try {
+ waves[year] = await api.fetchDistrictEstimates(district.leaid, { model: WAVE_MODEL, year })
+ } catch (err) {
+ console.error(`ChartPanel: wave ${year} fetch failed:`, err)
+ }
+ }),
+ )
+ if (cancelled) return
+ setWaveData(waves)
+ setLoading(false)
+ }
+
+ run()
+ return () => {
+ cancelled = true
+ }
+ }, [district.leaid])
+
+ // ——— Fetch: the current wave's summary for whichever spec is selected ———
+ // Only the selected specification is fetched. Enrollment and observed arrest
+ // counts are district facts, not model outputs, and the modelled distributions
+ // now come from real draws rather than from these rows — so prefetching all
+ // four specs' summaries would be four requests for data three of which are
+ // never read.
+ useEffect(() => {
+ let cancelled = false
+ if (summaryByModel[selectedModel]) return
+
+ async function run() {
try {
- // Chart 1 + 2: all 3 waves × three-year model (unified_m3_mod)
- const waveData = {}
- await Promise.all(ALL_WAVES.map(async (year) => {
- try { waveData[year] = await api.fetchDistrictEstimates(district.leaid, { model: WAVE_MODEL, year }) } catch (e) {}
- }))
-
- // Chart 3: all 4 quadrant models, so the dropdown can switch between them
- const quadData = {}
- await Promise.all(QUADRANT_MODELS.map(async (model) => {
- try { quadData[model] = await api.fetchDistrictEstimates(district.leaid, { model, year: '21-22' }) } catch (e) {}
- }))
-
- setData({ waveData, quadData })
+ const rows = await api.fetchDistrictEstimates(district.leaid, {
+ model: selectedModel,
+ year: CURRENT_WAVE,
+ })
+ if (!cancelled) setSummaryByModel((prev) => ({ ...prev, [selectedModel]: rows }))
} catch (err) {
- console.error('ChartPanel data fetch failed:', err)
- } finally {
- setLoading(false)
+ console.error(`ChartPanel: summary fetch failed for ${selectedModel}:`, err)
+ if (!cancelled) setSummaryByModel((prev) => ({ ...prev, [selectedModel]: [] }))
}
}
- fetchData()
- }, [district.leaid])
+ run()
+ return () => {
+ cancelled = true
+ }
+ }, [district.leaid, selectedModel, summaryByModel])
- if (loading || !data) {
- return
- }
+ const rows = useMemo(() => summaryByModel[selectedModel] || [], [summaryByModel, selectedModel])
- const mostRecent = data.waveData['21-22'] || []
+ const autoPooled = useMemo(() => shouldPoolBySex(rows), [rows])
+ const pooled = poolOverride ?? autoPooled
+
+ const groups = useMemo(() => buildDisplayGroups(rows, pooled), [rows, pooled])
+ const enrollByGroup = useMemo(() => enrollByGroupKey(rows), [rows])
+
+ const defaultKeys = useMemo(() => defaultSelectedKeys(groups), [groups])
+ const selectedKeys = selection?.pooled === pooled ? selection.keys : defaultKeys
+
+ const defaultPair = useMemo(() => defaultDiffPair(groups), [groups])
+ const pair = pairSelection?.pooled === pooled ? pairSelection.pair : defaultPair
+
+ const selectionIsEnrollmentFallback =
+ groups.length > 0 && groups.every((g) => g.observed === 0) && selection === null
+
+ const handleToggleGroup = useCallback(
+ (key) => {
+ const next = selectedKeys.includes(key)
+ ? selectedKeys.filter((k) => k !== key)
+ : [...selectedKeys, key]
+ setSelection({ pooled, keys: next })
+ },
+ [selectedKeys, pooled],
+ )
+
+ const handlePooledChange = useCallback((next) => {
+ setPoolOverride(next)
+ // Both selections live in the other namespace now — drop them and let the
+ // defaults recompute for the new group set.
+ setSelection(null)
+ setPairSelection(null)
+ }, [])
+
+ const handlePairChange = useCallback((next) => setPairSelection({ pooled, pair: next }), [pooled])
+
+ const drawModels = useMemo(
+ () => (compareAll ? QUADRANT_MODELS : [selectedModel]),
+ [compareAll, selectedModel],
+ )
+ const { status, byModel, nDraws } = useDrawDistribution({
+ leaid: district.leaid,
+ state,
+ models: drawModels,
+ year: CURRENT_WAVE,
+ })
+
+ if (loading || !waveData) return
- // Chart 1: Arrests over time by wave — observed total + modeled (three-year model) point-range
const timeSeriesData = ALL_WAVES.map((year) => {
- const yearRows = data.waveData[year] || []
+ const yearRows = waveData[year] || []
return {
year,
label: `20${year.replace('-', '-')}`,
@@ -64,20 +169,8 @@ export default function ChartPanel({ district, state }) {
}
})
- // Chart 2: rate by student group (most recent wave, per 1k), with modeled + observed + interval
- const rateByGroup = mostRecent.map((r) => ({
- race: r.race, sex: r.sex, label: groupLabel(r.race, r.sex),
- observedRate: (r.observed_arrests || 0) / ((r.stu_enroll || 1) / 1000),
- modeledMedian: (r.count_median || 0) / ((r.stu_enroll || 1) / 1000),
- rateLower: (r.rate_lower || 0) * 1000,
- rateUpper: (r.rate_upper || 0) * 1000,
- observedArrests: r.observed_arrests || 0,
- enrollment: r.stu_enroll || 0,
- }))
-
return (
- {/* Chart panel header */}
District estimates — Bayesian model comparison
@@ -85,29 +178,68 @@ export default function ChartPanel({ district, state }) {
-
+
+
+
+
+
+
+
-
-
-
-
- {/* Methodology footer */}
-
+
Methodology
- Estimates are from the CRDC School Arrest Rate API (Knowles & Miller 2025). Data shown spans three waves of
- the Civil Rights Data Collection (2015–16, 2017–18, 2021–22) and lets you explore four Bayesian model
- specifications: one-year vs. three-year models with and without referral-rate covariates. All rates are
- per 1,000 students.
+ Estimates are from the CRDC School Arrest Rate API (Knowles & Miller 2025). Data shown
+ spans three waves of the Civil Rights Data Collection (2015–16, 2017–18, 2021–22) and lets
+ you explore four Bayesian model specifications: one-year vs. three-year models with and
+ without referral-rate covariates. All rates are per 1,000 students. Modelled distributions
+ are drawn from the published posterior predictive draws, fetched in the browser.
diff --git a/src/components/DistrictSummaryTable.jsx b/src/components/DistrictSummaryTable.jsx
new file mode 100644
index 0000000..8bf7c53
--- /dev/null
+++ b/src/components/DistrictSummaryTable.jsx
@@ -0,0 +1,234 @@
+import { raceColor } from '../utils/colors.js'
+import { POOL_BY_SEX_ARREST_THRESHOLD } from '../utils/pooling.js'
+
+/**
+ * The results page's first block: what was actually observed, before any
+ * modelling. One row per student group plus a district total.
+ *
+ * The table doubles as the legend and the control for the density panel — each
+ * row carries the checkbox that adds or removes that group's curve, and the
+ * colour swatch is the same hue the curve is drawn in. That's deliberate:
+ * a separate legend plus a separate group picker would make the reader hold
+ * three mappings in their head instead of one.
+ *
+ * @param {{
+ * groups: Array<{key:string, race:string, sex:string|null, label:string,
+ * enroll:number, observed:number, rate:number}>,
+ * selectedKeys: string[],
+ * onToggleGroup: (key: string) => void,
+ * pooled: boolean,
+ * autoPooled: boolean,
+ * onPooledChange: (pooled: boolean) => void,
+ * selectionIsEnrollmentFallback: boolean,
+ * }} props
+ */
+export default function DistrictSummaryTable({
+ groups,
+ selectedKeys,
+ onToggleGroup,
+ pooled,
+ autoPooled,
+ onPooledChange,
+ selectionIsEnrollmentFallback,
+}) {
+ if (!groups?.length) {
+ return (
+
+
Reported arrests and enrollment
+
+ No student-group data is available for this district in 2021–22.
+
+ No group in this district reported an arrest in 2021–22, so the two largest groups by
+ enrollment are shown by default.
+
+ )}
+
+
+
+
+ Students counted are those in the four modelled race groups (American Indian / Alaska
+ Native, Black, Hispanic, White) — not the district’s total enrollment, which also
+ includes groups this model does not estimate.
+
+ {autoPooled ? (
+ <>
+ This district reported {totalObserved.toLocaleString()} arrests in total — fewer than{' '}
+ {POOL_BY_SEX_ARREST_THRESHOLD} — so Female and Male students are combined within each
+ race to give each estimate more data to stand on.
+
+ ) : (
+ <>Female and Male students are combined within each race.
+ )}
+
+
+
+ )
+}
+
+function formatRate(rate) {
+ if (!(rate > 0)) return '0.0'
+ return rate.toLocaleString(undefined, { minimumFractionDigits: 1, maximumFractionDigits: 1 })
+}
+
+const cardTitle = { fontSize: '0.85rem', marginBottom: 'var(--space-1)', color: 'var(--cv-ink-2)' }
+
+const tableStyle = {
+ width: '100%',
+ borderCollapse: 'collapse',
+ fontSize: '0.85rem',
+ fontVariantNumeric: 'tabular-nums',
+}
+
+const thStyle = {
+ textAlign: 'right',
+ padding: '0.35rem 0.5rem',
+ borderBottom: '1px solid var(--cv-rule-strong)',
+ fontSize: '0.72rem',
+ fontWeight: 600,
+ textTransform: 'uppercase',
+ letterSpacing: '0.06em',
+ color: 'var(--cv-ink-3)',
+ whiteSpace: 'nowrap',
+}
+
+const tdStyle = {
+ padding: '0.35rem 0.5rem',
+ borderBottom: '1px solid var(--cv-rule)',
+}
+
+const numStyle = { ...tdStyle, textAlign: 'right', whiteSpace: 'nowrap' }
+
+const bannerStyle = {
+ background: 'var(--cv-paper-2)',
+ border: '1px solid var(--cv-rule)',
+ borderLeft: '3px solid var(--cv-accent)',
+ borderRadius: 'var(--radius-md)',
+ padding: 'var(--space-1) var(--space-2)',
+ marginBottom: 'var(--space-2)',
+ display: 'flex',
+ flexDirection: 'column',
+ gap: '0.4rem',
+}
+
+const toggleLabel = {
+ display: 'inline-flex',
+ alignItems: 'center',
+ gap: '0.45rem',
+ fontSize: '0.78rem',
+ color: 'var(--cv-ink-2)',
+ cursor: 'pointer',
+}
+
+const noteStyle = {
+ fontSize: '0.78rem',
+ fontStyle: 'italic',
+ color: 'var(--cv-ink-3)',
+ margin: '0 0 var(--space-2)',
+}
diff --git a/src/hooks/useDrawDistribution.js b/src/hooks/useDrawDistribution.js
index 78e08d8..7ed4924 100644
--- a/src/hooks/useDrawDistribution.js
+++ b/src/hooks/useDrawDistribution.js
@@ -1,19 +1,68 @@
-import { useEffect, useState } from 'react'
+import { useEffect, useMemo, useState } from 'react'
import { getDb } from '../utils/duckdbClient.js'
-import { groupKey } from '../utils/drawGroups.js'
+import { groupKey, isCompleteDrawSet } from '../utils/drawGroups.js'
-const HF_BASE = 'https://huggingface.co/datasets/civilytics/crdc-school-arrest-rates/resolve/main/parquet'
+const HF_DATASET = 'civilytics/crdc-school-arrest-rates'
+const HF_BASE = `https://huggingface.co/datasets/${HF_DATASET}/resolve/main/parquet`
+const HF_TREE = `https://huggingface.co/api/datasets/${HF_DATASET}/tree/main/parquet`
-// Module-level cache: one registered duckdb-wasm file buffer per
+// Module-level cache: the registered duckdb-wasm file buffers for one
// (model, year, state) shard, shared across every component instance and
// district navigated to in this browser session. See Global Constraints —
-// in-memory only, no persistence across page loads.
+// in-memory only, no persistence across page loads. Four models is simply four
+// cache entries; nothing else about this cache changes when comparing specs.
const shardCache = new Map()
+// Runaway guard on a malformed directory listing, not an expected limit.
+const MAX_SHARD_PARTS = 64
+
function shardKey(model, year, state) {
return `${model}__${year}__${state}`
}
+function shardDir(model, year, state) {
+ return `model_id=${model}/YEAR=${year}/LEA_STATE=${state}`
+}
+
+/**
+ * Lists the parquet parts published for one (model, year, state).
+ *
+ * A state's draws are split across `data_0.parquet`, `data_1.parquet`, … and
+ * the part count varies by state: Nevada is one file, California is eight
+ * (6.2MB in total, of which data_0 is 37KB). Reading only data_0 covers 11 of
+ * California's 1,715 districts and makes every other CA district look absent
+ * from the published data, silently falling back to the approximation.
+ *
+ * The directory listing is used rather than probing `data_N` until a 404
+ * because a 404 is logged as a console error by the browser's network layer no
+ * matter how cleanly the fetch handles it — and a red error on every load is
+ * indistinguishable from a real one. Probing remains the fallback if the
+ * listing API is unavailable or changes shape.
+ */
+async function listShardParts(model, year, state) {
+ const dir = shardDir(model, year, state)
+ try {
+ const res = await fetch(`${HF_TREE}/${dir}`)
+ if (!res.ok) throw new Error(`tree listing HTTP ${res.status}`)
+ const entries = await res.json()
+ const parts = entries
+ .filter((e) => e?.type === 'file' && /\/data_\d+\.parquet$/.test(e.path || ''))
+ .map((e) => ({ part: Number(e.path.match(/data_(\d+)\.parquet$/)[1]), path: e.path }))
+ .sort((a, b) => a.part - b.part)
+ if (parts.length > 0) return parts.map((p) => p.part)
+ throw new Error('tree listing contained no parquet parts')
+ } catch (err) {
+ console.warn(`useDrawDistribution: falling back to sequential part probing for ${dir}:`, err.message)
+ const parts = []
+ for (let part = 0; part < MAX_SHARD_PARTS; part++) {
+ const res = await fetch(`${HF_BASE}/${dir}/data_${part}.parquet`, { method: 'HEAD' })
+ if (!res.ok) break
+ parts.push(part)
+ }
+ return parts
+ }
+}
+
function ensureShardRegistered(db, model, year, state) {
const key = shardKey(model, year, state)
if (!shardCache.has(key)) {
@@ -21,13 +70,24 @@ function ensureShardRegistered(db, model, year, state) {
key,
(async () => {
try {
- const url = `${HF_BASE}/model_id=${model}/YEAR=${year}/LEA_STATE=${state}/data_0.parquet`
- const res = await fetch(url)
- if (!res.ok) throw new Error(`Failed to fetch draw shard: HTTP ${res.status}`)
- const buffer = new Uint8Array(await res.arrayBuffer())
- const fileName = `${key}.parquet`
- await db.registerFileBuffer(fileName, buffer)
- return fileName
+ const dir = shardDir(model, year, state)
+ const parts = await listShardParts(model, year, state)
+ if (parts.length === 0) {
+ throw new Error(`No draw shard published for ${state}/${year}/${model}`)
+ }
+ // Parts are independent files, so fetch them together rather than
+ // walking them one at a time — California is eight round trips.
+ const fileNames = await Promise.all(
+ parts.map(async (part) => {
+ const res = await fetch(`${HF_BASE}/${dir}/data_${part}.parquet`)
+ if (!res.ok) throw new Error(`Failed to fetch draw shard part ${part}: HTTP ${res.status}`)
+ const buffer = new Uint8Array(await res.arrayBuffer())
+ const fileName = `${key}__${part}.parquet`
+ await db.registerFileBuffer(fileName, buffer)
+ return fileName
+ }),
+ )
+ return fileNames
} catch (err) {
// Don't let a transient failure (network blip, HF outage) poison the
// cache forever — remove the rejected entry so the next caller for
@@ -42,100 +102,145 @@ function ensureShardRegistered(db, model, year, state) {
}
/**
- * Fetches real posterior draws for one district/model/year from the Hugging
- * Face parquet dataset via duckdb-wasm, converts predicted counts to
- * rate-per-1,000 using each group's stu_enroll (not present in the draws
- * table itself — joined here from data this app already has), and returns
- * them keyed by "RACE_SEX" (see `groupKey` in utils/drawGroups.js).
+ * Reads one model's shard and returns that district's predicted counts keyed by
+ * group and indexed by draw.
*
- * `status` is an ANY-group signal: 'ready' means at least one group came back
- * with real draws, not that every requested group did. Groups can be missing
- * individually (falsy enrollment, or absent from the parquet shard), so a caller
- * that wants to claim "these are all real draws" must check its own rendered
- * groups against `drawsByGroup` — use `hasDrawsForAll` from utils/drawGroups.js.
- *
- * @param {{leaid: string, state: string, model: string, year: string,
- * groups: Array<{race: string, sex: string, stuEnroll: number}>}} params
- * @returns {{status: 'loading'|'ready'|'error', drawsByGroup: Record | null}}
+ * Indexing by `draw_id - 1` rather than by push order means DuckDB's row
+ * ordering is irrelevant, and it makes a missing draw a hole instead of a
+ * silently shorter array — which `isCompleteDrawSet` then rejects.
*/
-export function useDrawDistribution({ leaid, state, model, year, groups }) {
- const [status, setStatus] = useState('loading')
- const [drawsByGroup, setDrawsByGroup] = useState(null)
+async function fetchModelCounts(db, { leaid, state, model, year }) {
+ const fileNames = await ensureShardRegistered(db, model, year, state)
+ let conn
+ try {
+ conn = await db.connect()
+ // read_parquet over the full part list — a district lives in exactly one
+ // part, and which one is not predictable from its LEAID.
+ const fileList = fileNames.map((f) => `'${f}'`).join(', ')
+ const stmt = await conn.prepare(
+ `SELECT RACE, SEX, draw_id, pred FROM read_parquet([${fileList}]) WHERE LEAID = ?`,
+ )
+ const table = await stmt.query(leaid)
+ await stmt.close()
+ const rows = table.toArray().map((r) => r.toJSON())
- // groups is typically a fresh array literal every render; derive a stable
+ const raw = {}
+ let nDraws = 0
+ for (const row of rows) {
+ const drawId = Number(row.draw_id)
+ const pred = Number(row.pred)
+ if (!Number.isFinite(drawId) || drawId < 1 || !Number.isFinite(pred)) continue
+ const key = groupKey(row.RACE, row.SEX)
+ ;(raw[key] ??= [])[drawId - 1] = pred
+ if (drawId > nDraws) nDraws = drawId
+ }
+
+ // A group present for only part of the draw set must fall back, not render
+ // a short draw set: its density and interval would be computed off a
+ // biased subsample and look identical on screen to a complete one.
+ const counts = {}
+ for (const [key, arr] of Object.entries(raw)) {
+ if (isCompleteDrawSet(arr, nDraws)) counts[key] = arr
+ else console.warn('useDrawDistribution: dropping incomplete draw set for', { leaid, model, key, got: arr.length, want: nDraws })
+ }
+
+ return Object.keys(counts).length > 0 ? { counts, nDraws } : null
+ } finally {
+ if (conn) await conn.close()
+ }
+}
+
+/**
+ * Fetches real posterior predictive draws for one district from the Hugging
+ * Face parquet dataset via duckdb-wasm, for one or more model specifications at
+ * once, and returns **predicted counts indexed by draw** — not rates.
+ *
+ * Counts rather than rates is what unlocks the rest of the app: sex pooling has
+ * to sum numerators and denominators separately (see `utils/pooling.js`), and
+ * a between-group difference has to be taken at a common draw index. Callers
+ * divide by their own enrollment, which is why this hook no longer takes a
+ * `groups` argument at all.
+ *
+ * A caveat worth carrying into any caption: `draw_id` is renumbered 1–500 per
+ * write batch upstream, and a district's groups land in different batches, so
+ * draws are **not** paired parameter draws across groups — the pairing is
+ * effectively independent, dominated by `posterior_predict` observation noise.
+ * Published Fig 7 has the same property. Say "posterior predictive draws".
+ *
+ * `status` is an ANY-model, ANY-group signal: 'ready' means at least one model
+ * returned at least one complete group. A caller claiming "these are all real
+ * draws" must check its own rendered groups with `hasDrawsForAll`.
+ *
+ * @param {{leaid: string, state: string, models: string[], year: string}} params
+ * @returns {{status: 'loading'|'ready'|'error',
+ * byModel: Record, nDraws: number} | null> | null,
+ * nDraws: number}} `nDraws` is the largest draw count among models that came
+ * back; per-model counts live in `byModel[model].nDraws`.
+ */
+export function useDrawDistribution({ leaid, state, models, year }) {
+ const [status, setStatus] = useState('loading')
+ const [byModel, setByModel] = useState(null)
+
+ // `models` is typically a fresh array literal every render; derive a stable
// primitive so the effect only re-runs when its actual content changes.
- const groupsSignature = (groups || []).map((g) => `${g.race}:${g.sex}:${g.stuEnroll}`).join(',')
+ const modelsSignature = (models || []).filter(Boolean).join(',')
useEffect(() => {
let cancelled = false
// Reset BEFORE the input guard below, not after. Clearing any previous
// model/district's draws has to happen on every input change, including the
// ones that have nothing to fetch. Without this ordering, a chart that
- // varies `model` across renders (e.g. RateDensityRidgeline's dropdown) and
- // lands on a model whose `groups` is empty (that model's upstream fetch
- // failed) would keep reporting 'ready' and keep handing back the *previous*
- // model's real draws — keyed by the same RACE_SEX strings — under the newly
- // selected model's summary stats, silently mixing two models' data.
+ // varies its model selection across renders and lands on a model whose
+ // fetch fails would keep reporting 'ready' and keep handing back the
+ // *previous* model's real draws — keyed by the same group strings — under
+ // the newly selected model's label, silently mixing two models' data.
setStatus('loading')
- setDrawsByGroup(null)
+ setByModel(null)
+
+ const modelList = modelsSignature ? modelsSignature.split(',') : []
// Nothing to fetch: stay in 'loading' with no draws, which every consumer
// already treats as "fall back to the approximation". No cleanup needed —
// nothing async was started.
- if (!leaid || !state || !model || !year || !groups?.length) return
+ if (!leaid || !state || !year || modelList.length === 0) return
async function run() {
- let conn
try {
const db = await getDb()
- const fileName = await ensureShardRegistered(db, model, year, state)
- conn = await db.connect()
- const stmt = await conn.prepare(`SELECT RACE, SEX, pred FROM read_parquet('${fileName}') WHERE LEAID = ?`)
- const table = await stmt.query(leaid)
- await stmt.close()
- const rows = table.toArray().map((r) => r.toJSON())
+ // Per-model try/catch: one shard 404ing (or one model missing for this
+ // state) must not blank out the models that did load.
+ const results = await Promise.all(
+ modelList.map(async (model) => {
+ try {
+ return [model, await fetchModelCounts(db, { leaid, state, model, year })]
+ } catch (err) {
+ console.error(`useDrawDistribution: model ${model} failed:`, err)
+ return [model, null]
+ }
+ }),
+ )
- const enrollByGroup = {}
- for (const g of groups) enrollByGroup[groupKey(g.race, g.sex)] = g.stuEnroll || 0
-
- const byGroup = {}
- for (const row of rows) {
- const key = groupKey(row.RACE, row.SEX)
- const enroll = enrollByGroup[key]
- if (!enroll) continue
- const rate = (Number(row.pred) / enroll) * 1000
- ;(byGroup[key] ??= []).push(rate)
+ if (cancelled) return
+ const next = Object.fromEntries(results)
+ const anyReady = Object.values(next).some((v) => v !== null)
+ if (anyReady) {
+ setByModel(next)
+ setStatus('ready')
+ } else {
+ console.warn('useDrawDistribution: no usable draws for', { leaid, state, year, models: modelList })
+ setByModel(null)
+ setStatus('error')
}
-
- // A successful query with zero matching rows (this district isn't in
- // the draws shard, or none of its rows matched a known group) is not
- // "ready" — there's no real data to show, so treat it like a failure
- // and let the caller fall back, rather than silently claiming real
- // draws while every group actually uses the fitted approximation.
- const hasDraws = Object.values(byGroup).some((draws) => draws.length > 0)
-
- if (!cancelled) {
- if (hasDraws) {
- setDrawsByGroup(byGroup)
- setStatus('ready')
- } else {
- console.warn('useDrawDistribution: query succeeded but returned no usable draws for', { leaid, model, year, state })
- setDrawsByGroup(null)
- setStatus('error')
- }
- }
} catch (err) {
+ // Only reached when the shared duckdb engine itself fails to load.
console.error('useDrawDistribution failed:', err)
if (!cancelled) {
- // Belt-and-suspenders alongside the setDrawsByGroup(null) at the
- // top of this effect: a failed fetch/query must never leave a
- // *previous* model's real draws in place under the newly-selected
- // model's status/summary stats.
- setDrawsByGroup(null)
+ // Belt-and-suspenders alongside the setByModel(null) at the top of
+ // this effect: a failed load must never leave a *previous* model's
+ // real draws in place under the newly-selected model's label.
+ setByModel(null)
setStatus('error')
}
- } finally {
- if (conn) await conn.close()
}
}
@@ -143,8 +248,12 @@ export function useDrawDistribution({ leaid, state, model, year, groups }) {
return () => {
cancelled = true
}
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [leaid, state, model, year, groupsSignature])
+ }, [leaid, state, year, modelsSignature])
- return { status, drawsByGroup }
+ const nDraws = useMemo(
+ () => Math.max(0, ...Object.values(byModel || {}).map((v) => v?.nDraws || 0)),
+ [byModel],
+ )
+
+ return { status, byModel, nDraws }
}
diff --git a/src/styles/tokens.css b/src/styles/tokens.css
index c3ee0f1..c6d5331 100644
--- a/src/styles/tokens.css
+++ b/src/styles/tokens.css
@@ -270,3 +270,9 @@ code, .mono { font-family: var(--font-mono); font-feature-settings: "tnum" 1, "z
/* Responsive */
@media (max-width: 680px) { .cv-footer { flex-direction: column; text-align: center; } }
+
+/* Spinner used while opt-in multi-shard draw fetches are in flight */
+@keyframes cv-spin { to { transform: rotate(360deg); } }
+@media (prefers-reduced-motion: reduce) {
+ @keyframes cv-spin { to { transform: none; } }
+}
diff --git a/src/utils/agrestiCoull.js b/src/utils/agrestiCoull.js
new file mode 100644
index 0000000..7cc83f0
--- /dev/null
+++ b/src/utils/agrestiCoull.js
@@ -0,0 +1,66 @@
+/**
+ * Agresti–Coull approximate interval for a rare-event count.
+ *
+ * Direct port of `agresti_coull()` in crdc-arrests/R/paper_figures.R:219-237 —
+ * the frequentist point range drawn beside the posterior densities in the white
+ * paper's Figs 6 and 7. Keeping this a faithful port is the whole point: the
+ * app's error bars have to be the *same* interval the paper published, so the
+ * two can be compared directly.
+ *
+ * Two consequences of that faithfulness, both deliberate:
+ *
+ * 1. The bounds are on the **count** scale, not the proportion scale — the R
+ * function multiplies back up by the adjusted denominator. Divide by
+ * enrollment yourself to plot per-1,000.
+ * 2. `lower` can be **negative** for very small numerators (e.g. 1 arrest in 53
+ * students gives -0.32). Clamp for display at the call site; do not clamp
+ * here, or this stops matching the published figures.
+ *
+ * The R original returns an unnamed vector in the order
+ * `c(ci_upper, ci_lower, sd, phat_se, phat)` — upper *first*, which is easy to
+ * transcribe backwards. This returns a named object instead.
+ */
+
+import { probit } from './distributionApprox.js'
+
+/**
+ * @param {number} numerator - observed events (arrests)
+ * @param {number} denominator - trials (students enrolled)
+ * @param {number} [confidenceLevel=0.95] - e.g. 0.95 for a 95% interval
+ * @returns {{upper: number, lower: number, sd: number, se: number, phat: number}}
+ * `upper`/`lower`/`sd` are counts. In the zero-numerator branch the R
+ * original also reports `phat` as a count (the interval midpoint) rather than
+ * a proportion; that quirk is preserved.
+ */
+export function agrestiCoull(numerator, denominator, confidenceLevel = 0.95) {
+ const adjStar = probit(1 - (1 - confidenceLevel) / 2)
+
+ if (numerator > 0) {
+ const numStar = numerator + adjStar
+ const denomStar = denominator + 2 * adjStar
+ const phat = numStar / denomStar
+ const se = Math.sqrt((phat / denomStar) * (1 - phat))
+ return {
+ upper: (phat + adjStar * se) * denomStar,
+ lower: (phat - adjStar * se) * denomStar,
+ sd: se * denomStar,
+ se,
+ phat,
+ }
+ }
+
+ // Zero events: the rule of three. The R original writes the bound as
+ // denominator * (-log(1 - level) / denominator), where the denominator
+ // cancels — so the upper bound is -log(1 - level) ≈ 3 at 95% regardless of
+ // how many students were enrolled. Kept in the cancelled form so the value
+ // is identical rather than merely close.
+ const upper = -Math.log(1 - confidenceLevel)
+ const midpoint = (upper + 0) / 2
+ return {
+ upper,
+ lower: 0,
+ sd: midpoint / confidenceLevel,
+ se: 0,
+ phat: midpoint,
+ }
+}
diff --git a/src/utils/agrestiCoull.test.js b/src/utils/agrestiCoull.test.js
new file mode 100644
index 0000000..d1c477b
--- /dev/null
+++ b/src/utils/agrestiCoull.test.js
@@ -0,0 +1,109 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { agrestiCoull } from './agrestiCoull.js'
+
+/**
+ * Reference values produced by the R original, `agresti_coull()` in
+ * crdc-arrests/R/paper_figures.R:219-237, printed at 15 significant digits:
+ *
+ * agresti_coull(15, 499, 0.95) c(24.89431440404100, 9.02561356503911,
+ * 4.04821235598516, 0.00804941727469993,
+ * 0.03372299056239180)
+ * agresti_coull(0, 53, 0.95) c(2.99573227355399, 0, 1.57670119660736,
+ * 0, 1.49786613677699)
+ * agresti_coull(1, 53, 0.95) c(6.24314599198645, -0.32321802290634,
+ * 1.67512364173205, 0.02942947578293,
+ * 0.05200224403214)
+ * agresti_coull(100, 148928, 0.95) c(121.743969120453, 82.1759588486273,
+ * 10.0940656522091, 6.77763749845643e-05,
+ * 6.84607866694077e-04)
+ * agresti_coull(3, 200, 0.90) c(8.14909680812749, 1.14061044577545,
+ * 2.13042858267619, 0.01047976610058,
+ * 0.02284844466400)
+ *
+ * R's qnorm is exact to double precision; this port uses Acklam's rational
+ * probit (relative error < 1.15e-9), so equality is asserted to 1e-7 relative.
+ */
+const REL_TOL = 1e-7
+
+function assertClose(actual, expected, label) {
+ const scale = Math.max(Math.abs(expected), 1e-9)
+ assert.ok(
+ Math.abs(actual - expected) / scale < REL_TOL,
+ `${label}: expected ${expected}, got ${actual}`,
+ )
+}
+
+function assertMatchesR(result, [upper, lower, sd, se, phat], label) {
+ assertClose(result.upper, upper, `${label} upper`)
+ assertClose(result.lower, lower, `${label} lower`)
+ assertClose(result.sd, sd, `${label} sd`)
+ assertClose(result.se, se, `${label} se`)
+ assertClose(result.phat, phat, `${label} phat`)
+}
+
+test('agrestiCoull: matches R for a typical rare-event cell', () => {
+ assertMatchesR(
+ agrestiCoull(15, 499),
+ [24.894314404041, 9.02561356503911, 4.04821235598516, 0.00804941727469993, 0.0337229905623918],
+ 'ac(15, 499, 0.95)',
+ )
+})
+
+test('agrestiCoull: matches R for a large district cell', () => {
+ assertMatchesR(
+ agrestiCoull(100, 148928),
+ [121.743969120453, 82.1759588486273, 10.0940656522091, 6.77763749845643e-5, 6.84607866694077e-4],
+ 'ac(100, 148928, 0.95)',
+ )
+})
+
+test('agrestiCoull: matches R at a non-default confidence level', () => {
+ assertMatchesR(
+ agrestiCoull(3, 200, 0.9),
+ [8.14909680812749, 1.14061044577545, 2.13042858267619, 0.0104797661005796, 0.0228484446639996],
+ 'ac(3, 200, 0.90)',
+ )
+})
+
+test('agrestiCoull: zero numerator uses the rule of three', () => {
+ // -log(1 - 0.95) = 2.9957…, the classic "rule of three" upper bound for zero
+ // events. The R original writes it as denominator * (-log(1-cl)/denominator),
+ // which cancels — the bound does not depend on the denominator at all.
+ const r = agrestiCoull(0, 53)
+ assertMatchesR(r, [2.99573227355399, 0, 1.57670119660736, 0, 1.49786613677699], 'ac(0, 53, 0.95)')
+ assert.equal(r.lower, 0)
+})
+
+test('agrestiCoull: the zero-numerator upper bound ignores the denominator', () => {
+ assert.equal(agrestiCoull(0, 53).upper, agrestiCoull(0, 500000).upper)
+})
+
+test('agrestiCoull: keeps R\'s negative lower bound for a single event', () => {
+ // Faithful to the R original: with numerator = 1 the lower bound goes below
+ // zero. Callers clamp for display — do NOT clamp here, or this port silently
+ // stops matching the published figures.
+ const r = agrestiCoull(1, 53)
+ assertMatchesR(
+ r,
+ [6.24314599198645, -0.323218022906343, 1.67512364173205, 0.029429475782928, 0.0520022440321421],
+ 'ac(1, 53, 0.95)',
+ )
+ assert.ok(r.lower < 0, 'lower bound should be negative for numerator = 1, n = 53')
+})
+
+test('agrestiCoull: bounds are on the count scale, not the proportion scale', () => {
+ const r = agrestiCoull(15, 499)
+ assert.ok(r.upper > 15 && r.lower < 15, 'the interval should bracket the observed count')
+})
+
+test('agrestiCoull: defaults to 95%', () => {
+ assert.deepEqual(agrestiCoull(15, 499), agrestiCoull(15, 499, 0.95))
+})
+
+test('agrestiCoull: a wider confidence level gives a wider interval', () => {
+ const narrow = agrestiCoull(15, 499, 0.8)
+ const wide = agrestiCoull(15, 499, 0.99)
+ assert.ok(wide.upper > narrow.upper)
+ assert.ok(wide.lower < narrow.lower)
+})
diff --git a/src/utils/colors.js b/src/utils/colors.js
index 40ae8f8..4393030 100644
--- a/src/utils/colors.js
+++ b/src/utils/colors.js
@@ -31,10 +31,26 @@ export function raceColor(race) {
return RACE_COLORS[race] || REFERENCE_GRAY
}
+// Both label helpers take an optional `sex`: omitting it names a sex-pooled
+// group (race alone), mirroring `groupKey(race)` in utils/drawGroups.js. Don't
+// let a missing sex fall through to "Male".
export function groupLabel(race, sex) {
- return `${RACE_LABELS[race] || race} ${sex === 'F' ? 'Female' : 'Male'}`
+ const race_ = RACE_LABELS[race] || race
+ if (!sex) return race_
+ return `${race_} ${sex === 'F' ? 'Female' : 'Male'}`
}
export function shortGroupLabel(race, sex) {
- return `${SHORT_RACE_LABEL[race] || race} ${sex}`
+ const race_ = SHORT_RACE_LABEL[race] || race
+ return sex ? `${race_} ${sex}` : race_
+}
+
+// For labels that appear mid-sentence ("…the Black male arrest rate exceeds the
+// White male rate"). The race is a proper noun and keeps its capital; only the
+// sex word is lowercased. Don't reach for .toLowerCase() on groupLabel() — it
+// turns "Black Male" into "black male".
+export function sentenceGroupLabel(race, sex) {
+ const race_ = RACE_LABELS[race] || race
+ if (!sex) return race_
+ return `${race_} ${sex === 'F' ? 'female' : 'male'}`
}
diff --git a/src/utils/densityProfile.js b/src/utils/densityProfile.js
new file mode 100644
index 0000000..fe08e0f
--- /dev/null
+++ b/src/utils/densityProfile.js
@@ -0,0 +1,114 @@
+/**
+ * Chooses an honest visual profile for one group's posterior predictive draws.
+ *
+ * In a sparse district the posterior predictive is a *discrete count*
+ * distribution, not a smooth one. Carson City NV (3200390) has a 53-student
+ * AI/AN female cell where a single arrest is 18.9 per 1,000: the draws take
+ * four distinct values, and a Gaussian KDE renders those four spikes as a lumpy
+ * smear that reads as a rendering bug rather than as a finding about the data.
+ *
+ * So: few distinct values → draw the actual probability mass at each achievable
+ * rate. Many → the smooth KDE, delegated to `kde.js` unchanged (its bandwidth
+ * clamp was tuned for exactly these zero-inflated posteriors — see
+ * BANDWIDTH_FLOOR_DIVISOR there; do not re-derive it here).
+ */
+
+import { kdeCurve } from './kde.js'
+import { toRates } from './pooling.js'
+
+/**
+ * At or below this many distinct predicted counts, the draws are shown as
+ * discrete mass rather than smoothed.
+ *
+ * 12 is comfortably above the 3–5 distinct values a genuinely sparse cell
+ * produces and comfortably below the ~40+ a district with real arrest volume
+ * produces, so the switch happens well away from either regime rather than
+ * flickering at the boundary.
+ */
+export const MASS_MAX_DISTINCT = 12
+
+/**
+ * @param {number[] | null | undefined} counts - predicted counts, indexed by draw
+ * @param {number | null | undefined} enroll - students in the group
+ * @param {{min?: number, max?: number, n?: number}} [domain] - the x-range the
+ * profile will be drawn over, in rate per 1,000. Passed straight to
+ * `kdeCurve`, whose bandwidth clamp is relative to this width.
+ * @returns {{kind: 'kde'|'mass', points: Array<{x: number, y: number}>,
+ * maxY: number, step: number} | null}
+ * `null` when there is nothing to draw (no draws, or no denominator).
+ * For `kind: 'mass'`, `y` is a probability and `step` is the spacing between
+ * achievable rates — the bar width for a filled staircase. For `kind: 'kde'`,
+ * `y` is a density and `step` is 0.
+ * `maxY` is the profile's own peak: because the two kinds carry different y
+ * units, a chart overlaying several groups must normalize each profile by its
+ * own `maxY` rather than by a shared maximum.
+ */
+export function densityProfile(counts, enroll, domain = {}) {
+ const rates = toRates(counts, enroll)
+ if (!rates.length) return null
+ // Counts are integers, so consecutive achievable rates are exactly one
+ // student-rate apart — pass that explicitly rather than inferring it from the
+ // gaps actually observed, which overstates the bar width when (say) only
+ // counts 0 and 3 appear.
+ return rateProfile(rates, domain, 1000 / enroll)
+}
+
+/**
+ * The same choice made directly on a set of rates, for quantities that are
+ * already differences rather than count/denominator pairs (see
+ * `utils/groupDifference.js`). A difference of two discrete posteriors is
+ * itself discrete, and deserves the same honesty.
+ *
+ * @param {number[] | null | undefined} rates - values on the plotted scale
+ * @param {{min?: number, max?: number, n?: number}} [domain]
+ * @param {number} [explicitStep] - known spacing between achievable values;
+ * inferred from the smallest observed gap when omitted.
+ * @returns {{kind: 'kde'|'mass', points: Array<{x: number, y: number}>,
+ * maxY: number, step: number} | null}
+ */
+export function rateProfile(rates, domain = {}, explicitStep = 0) {
+ if (!rates?.length) return null
+
+ const { min = 0, max, n = 60 } = domain
+
+ // Round before tallying so two draws that differ only in floating-point noise
+ // count as one achievable value rather than two.
+ const tally = new Map()
+ for (const r of rates) {
+ const key = Math.round(r * 1e9) / 1e9
+ tally.set(key, (tally.get(key) || 0) + 1)
+ }
+
+ if (tally.size <= MASS_MAX_DISTINCT) {
+ const total = rates.length
+ const points = [...tally.entries()]
+ .map(([x, freq]) => ({ x, y: freq / total }))
+ .sort((a, b) => a.x - b.x)
+ return {
+ kind: 'mass',
+ points,
+ maxY: Math.max(...points.map((p) => p.y)),
+ step: explicitStep > 0 ? explicitStep : inferStep(points, max, min),
+ }
+ }
+
+ const points = kdeCurve(rates, { min, max, n })
+ return {
+ kind: 'kde',
+ points,
+ maxY: Math.max(...points.map((p) => p.y)),
+ step: 0,
+ }
+}
+
+/** Smallest gap between achievable values; a visible default for a single point. */
+function inferStep(points, max, min) {
+ let smallest = Infinity
+ for (let i = 1; i < points.length; i++) {
+ const gap = points[i].x - points[i - 1].x
+ if (gap > 0 && gap < smallest) smallest = gap
+ }
+ if (Number.isFinite(smallest)) return smallest
+ const width = Number.isFinite(max) && max > min ? max - min : 1
+ return width / 40
+}
diff --git a/src/utils/densityProfile.test.js b/src/utils/densityProfile.test.js
new file mode 100644
index 0000000..27c1142
--- /dev/null
+++ b/src/utils/densityProfile.test.js
@@ -0,0 +1,135 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { MASS_MAX_DISTINCT, densityProfile, rateProfile } from './densityProfile.js'
+import { kdeCurve } from './kde.js'
+import { toRates } from './pooling.js'
+
+/** counts whose distinct-value count is exactly `k` (values 0…k-1, padded). */
+function countsWithDistinct(k, total = 500) {
+ const out = []
+ for (let i = 0; i < total; i++) out.push(i % k)
+ return out
+}
+
+test('densityProfile: discrete posterior returns a mass profile', () => {
+ // 500 draws over 4 achievable counts is the Carson City AI/AN case: a KDE
+ // renders it as a lumpy smear that reads as a rendering bug.
+ const profile = densityProfile(countsWithDistinct(4), 53, { max: 100 })
+ assert.equal(profile.kind, 'mass')
+})
+
+test('densityProfile: mass points are probabilities at achievable rates', () => {
+ const profile = densityProfile([0, 0, 0, 1], 500, { max: 10 })
+ assert.equal(profile.kind, 'mass')
+ assert.deepEqual(profile.points, [
+ { x: 0, y: 0.75 },
+ { x: 2, y: 0.25 },
+ ])
+})
+
+test('densityProfile: mass probabilities sum to 1', () => {
+ const profile = densityProfile([0, 0, 1, 2, 2, 5], 1000, { max: 10 })
+ const total = profile.points.reduce((sum, p) => sum + p.y, 0)
+ assert.ok(Math.abs(total - 1) < 1e-12, `probabilities summed to ${total}`)
+})
+
+test('densityProfile: mass points are sorted by rate ascending', () => {
+ const profile = densityProfile([5, 0, 3, 1], 1000, { max: 10 })
+ const xs = profile.points.map((p) => p.x)
+ assert.deepEqual(xs, [...xs].sort((a, b) => a - b))
+})
+
+test('densityProfile: mass carries the achievable-rate step for staircase width', () => {
+ // Counts are integers, so achievable rates are spaced 1000/enroll apart.
+ // The chart needs that width to draw a bar rather than a hairline.
+ const profile = densityProfile([0, 1], 250, { max: 10 })
+ assert.equal(profile.step, 4)
+})
+
+test('densityProfile: a fully degenerate draw set is a single mass point', () => {
+ // 500 identical zeros is common in small districts. This is the case a
+ // Gaussian KDE turns into a delta spike.
+ const profile = densityProfile(new Array(500).fill(0), 4073, { max: 20 })
+ assert.equal(profile.kind, 'mass')
+ assert.deepEqual(profile.points, [{ x: 0, y: 1 }])
+})
+
+test('densityProfile: switches to KDE above the distinct-value cutoff', () => {
+ const atCutoff = densityProfile(countsWithDistinct(MASS_MAX_DISTINCT), 1000, { max: 50 })
+ const aboveCutoff = densityProfile(countsWithDistinct(MASS_MAX_DISTINCT + 1), 1000, { max: 50 })
+ assert.equal(atCutoff.kind, 'mass')
+ assert.equal(aboveCutoff.kind, 'kde')
+})
+
+test('densityProfile: KDE branch delegates to kdeCurve over the given domain', () => {
+ // The bandwidth clamp in kde.js was tuned for exactly these zero-inflated
+ // posteriors — this must delegate, not re-derive.
+ const counts = countsWithDistinct(40)
+ const enroll = 1000
+ const profile = densityProfile(counts, enroll, { min: 0, max: 50, n: 60 })
+ const expected = kdeCurve(toRates(counts, enroll), { min: 0, max: 50, n: 60 })
+ assert.equal(profile.kind, 'kde')
+ assert.deepEqual(profile.points, expected)
+})
+
+test('densityProfile: reports the profile peak for per-group normalization', () => {
+ const profile = densityProfile([0, 0, 0, 1], 500, { max: 10 })
+ assert.equal(profile.maxY, 0.75)
+
+ const kde = densityProfile(countsWithDistinct(40), 1000, { max: 50 })
+ assert.equal(kde.maxY, Math.max(...kde.points.map((p) => p.y)))
+})
+
+test('densityProfile: null for unusable input rather than an empty curve', () => {
+ assert.equal(densityProfile([], 500, { max: 10 }), null)
+ assert.equal(densityProfile(undefined, 500, { max: 10 }), null)
+ assert.equal(densityProfile([0, 1], 0, { max: 10 }), null)
+ assert.equal(densityProfile([0, 1], undefined, { max: 10 }), null)
+})
+
+test('densityProfile: does not mutate its input counts', () => {
+ const counts = [3, 1, 2]
+ densityProfile(counts, 1000, { max: 10 })
+ assert.deepEqual(counts, [3, 1, 2])
+})
+
+// ——— rateProfile (used for already-differenced quantities) ———
+
+test('rateProfile: mass profile straight from rate values', () => {
+ const profile = rateProfile([-1, -1, 0, 2], { min: -5, max: 5 })
+ assert.equal(profile.kind, 'mass')
+ assert.deepEqual(profile.points, [
+ { x: -1, y: 0.5 },
+ { x: 0, y: 0.25 },
+ { x: 2, y: 0.25 },
+ ])
+})
+
+test('rateProfile: infers the staircase step from the smallest observed gap', () => {
+ assert.equal(rateProfile([0, 0.5, 2], { min: 0, max: 5 }).step, 0.5)
+})
+
+test('rateProfile: falls back to a visible step for a single achievable value', () => {
+ const profile = rateProfile([3, 3, 3], { min: 0, max: 40 })
+ assert.ok(profile.step > 0, 'a single mass point still needs a drawable width')
+})
+
+test('rateProfile: an explicit step wins over the inferred one', () => {
+ assert.equal(rateProfile([0, 3], { min: 0, max: 5 }, 0.25).step, 0.25)
+})
+
+test('rateProfile: merges values differing only by floating-point noise', () => {
+ const profile = rateProfile([0.1 + 0.2, 0.3, 0.3], { min: 0, max: 1 })
+ assert.equal(profile.points.length, 1)
+ assert.equal(profile.points[0].y, 1)
+})
+
+test('rateProfile: switches to KDE above the distinct-value cutoff', () => {
+ const many = Array.from({ length: 300 }, (_, i) => (i % (MASS_MAX_DISTINCT + 1)) * 0.7)
+ assert.equal(rateProfile(many, { min: 0, max: 10 }).kind, 'kde')
+})
+
+test('rateProfile: null for an empty or missing set', () => {
+ assert.equal(rateProfile([], { max: 5 }), null)
+ assert.equal(rateProfile(undefined, { max: 5 }), null)
+})
diff --git a/src/utils/districtGroups.js b/src/utils/districtGroups.js
new file mode 100644
index 0000000..291dca9
--- /dev/null
+++ b/src/utils/districtGroups.js
@@ -0,0 +1,128 @@
+/**
+ * Turns the API's per-race×sex estimate rows into the row set the results page
+ * actually displays, in one place, so the summary table, the density panel and
+ * the difference chart can never disagree about which groups exist, what they
+ * are called, or which are selected by default.
+ *
+ * "Display space" is either eight race×sex groups or — when sex pooling is on —
+ * four race groups. Keys come from `groupKey`, so the same map lookups work in
+ * both modes.
+ */
+
+import { groupLabel, sentenceGroupLabel, shortGroupLabel } from './colors.js'
+import { groupKey } from './drawGroups.js'
+import { poolBySex } from './pooling.js'
+
+/** The four modeled race groups, in the fixed palette order (see colors.js). */
+export const RACE_ORDER = ['WH', 'BL', 'HI', 'AM']
+const SEXES = ['F', 'M']
+
+/**
+ * @param {Array