Compare commits
7
Commits
4396a56a4a
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
587e74ca80
|
||
|
|
654b42ca71
|
||
|
|
fb0beaafc8
|
||
|
|
15f8a86409
|
||
|
|
9df4f4b5aa | ||
|
|
fd7acddeb2
|
||
|
|
c8757cd786
|
@@ -170,8 +170,8 @@ time, the port does not).
|
|||||||
posterior predictive is drawn as discrete mass, because it *is* discrete; a
|
posterior predictive is drawn as discrete mass, because it *is* discrete; a
|
||||||
Gaussian KDE over four achievable values renders as a lumpy smear that reads
|
Gaussian KDE over four achievable values renders as a lumpy smear that reads
|
||||||
as a rendering bug.
|
as a rendering bug.
|
||||||
- **`src/utils/rateDomain.js`** — `MAX_RATE_DOMAIN = 30` caps the axis so a
|
- **`src/utils/rateDomain.js`** — `MAX_RATE_DOMAIN = 100` caps the axis so a
|
||||||
four-student cell can't squash every other curve. It reports `clipped` so the
|
four-student cell can't squash every other curve (up from 30, which was exceeded by over 60% of districts). It reports `clipped` so the
|
||||||
chart says so instead of silently cropping.
|
chart says so instead of silently cropping.
|
||||||
|
|
||||||
## Common Pitfalls & Gotchas
|
## Common Pitfalls & Gotchas
|
||||||
|
|||||||
@@ -0,0 +1,392 @@
|
|||||||
|
/**
|
||||||
|
* Analysis: distribution of maximum x-axis values for the rate density chart.
|
||||||
|
*
|
||||||
|
* The "Arrest rate probability density" chart currently caps its x-axis at a
|
||||||
|
* hard-coded MAX_RATE_DOMAIN = 30 (per 1,000 students). We want to understand
|
||||||
|
* what the true distribution of max x-values is across districts so we can
|
||||||
|
* decide whether to raise or make this cap dynamic.
|
||||||
|
*
|
||||||
|
* What feeds computeRateDomain():
|
||||||
|
* 1. Per-group rate arrays from real posterior draws → 0.995 quantile of each
|
||||||
|
* (requires parquet data; approximated here via model estimates)
|
||||||
|
* 2. Agresti-Coull upper bound rates = (ac.upper / enroll) * 1000, computed
|
||||||
|
* from observed counts and enrollment — available via the API for every
|
||||||
|
* race×sex cell in every district
|
||||||
|
*
|
||||||
|
* KEY INSIGHT: Only SELECTED groups feed into computeRateDomain. Per
|
||||||
|
* `defaultSelectedKeys` in districtGroups.js, only groups with ≥1 observed
|
||||||
|
* arrest are selected by default (or top-2 by enrollment if none have arrests).
|
||||||
|
* Additionally, when total district arrests < 20, sex pooling merges F+M cells,
|
||||||
|
* which combines enrollment and observed counts per race.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const BASE = 'https://crdc-api.civilytics.org/api/v1'
|
||||||
|
const LIMIT = 500
|
||||||
|
const POOL_THRESHOLD = 20 // POOL_BY_SEX_ARREST_THRESHOLD from pooling.js
|
||||||
|
|
||||||
|
/** Fetch JSON from the API, retrying on transient failures. */
|
||||||
|
async function apiFetch(path) {
|
||||||
|
const url = `${BASE}${path}`
|
||||||
|
let lastErr
|
||||||
|
for (let attempt = 0; attempt <= 3; attempt++) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, { signal: AbortSignal.timeout(30000) })
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||||
|
return await res.json()
|
||||||
|
} catch (err) {
|
||||||
|
lastErr = err
|
||||||
|
if (attempt < 3) {
|
||||||
|
const wait = 500 * Math.pow(2, attempt) + Math.random() * 100
|
||||||
|
console.error(` retry ${attempt + 1}/3 after ${Math.round(wait)}ms:`, err.message)
|
||||||
|
await new Promise((r) => setTimeout(r, wait))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Peter Acklam's inverse normal CDF (probit), ~1.15e-9 relative error. */
|
||||||
|
function probit(p) {
|
||||||
|
const a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02,
|
||||||
|
1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]
|
||||||
|
const b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02,
|
||||||
|
6.680131188771972e+01, -1.328068155288572e+01]
|
||||||
|
const c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00,
|
||||||
|
-2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]
|
||||||
|
const d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00,
|
||||||
|
3.754408661907416e+00]
|
||||||
|
const pLow = 0.02425, pHigh = 1 - pLow
|
||||||
|
|
||||||
|
if (p < pLow) {
|
||||||
|
const q = Math.sqrt(-2 * Math.log(p))
|
||||||
|
return (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /
|
||||||
|
((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
|
||||||
|
}
|
||||||
|
if (p <= pHigh) {
|
||||||
|
const q = p - 0.5, r = q * q
|
||||||
|
return (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5]) * q /
|
||||||
|
(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1)
|
||||||
|
}
|
||||||
|
const q = Math.sqrt(-2 * Math.log(1 - p))
|
||||||
|
return -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /
|
||||||
|
((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Agresti-Coull upper bound (count scale), port of agrestiCoull.js. */
|
||||||
|
function acUpperBound(numerator, denominator, confidenceLevel = 0.95) {
|
||||||
|
const adjStar = probit(1 - (1 - confidenceLevel) / 2)
|
||||||
|
if (numerator > 0) {
|
||||||
|
const numStar = numerator + adjStar
|
||||||
|
const denomStar = denominator + 2 * adjStar
|
||||||
|
const phat = numStar / denomStar
|
||||||
|
const se = Math.sqrt((phat / denomStar) * (1 - phat))
|
||||||
|
return (phat + adjStar * se) * denomStar
|
||||||
|
}
|
||||||
|
// Zero events: rule of three — upper bound ≈ 3 regardless of enrollment.
|
||||||
|
return -Math.log(1 - confidenceLevel)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Estimate the 0.995 quantile of posterior predictive draw rates. */
|
||||||
|
function estimateDrawQuantile(row, targetP = 0.995) {
|
||||||
|
const enroll = row.stu_enroll || 0
|
||||||
|
if (enroll <= 0) return null
|
||||||
|
|
||||||
|
// When count_upper is available and > 0, the model's posterior predictive
|
||||||
|
// upper bound gives a sense of the spread. For sparse groups with few or zero
|
||||||
|
// arrests, draw quantiles can be extreme because observation noise dominates:
|
||||||
|
// a single predicted arrest in a small cell produces a huge per-1000 rate.
|
||||||
|
const countUpper = row.count_upper || 0
|
||||||
|
if (countUpper > 0) {
|
||||||
|
return (countUpper / enroll) * 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
// When count_upper is 0, use the Agresti-Coull upper bound as a conservative
|
||||||
|
// proxy — it's an honest frequentist interval that also tends to be extreme
|
||||||
|
// for sparse groups.
|
||||||
|
const ac = acUpperBound(row.observed_arrests || 0, enroll)
|
||||||
|
return (ac / enroll) * 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Determine which race×sex cells are "selected" by default per districtGroups.js logic. */
|
||||||
|
function determineSelectedCells(cells, pooled) {
|
||||||
|
const usable = cells.filter(
|
||||||
|
(r) => ['WH', 'BL', 'HI', 'AM'].includes(r.race) && ['F', 'M'].includes(r.sex),
|
||||||
|
)
|
||||||
|
|
||||||
|
if (pooled) {
|
||||||
|
// When pooled: merge F+M per race, then select groups with ≥1 arrest or top-2 by enrollment
|
||||||
|
const races = {}
|
||||||
|
for (const r of usable) {
|
||||||
|
if (!races[r.race]) races[r.race] = { race: r.race, enroll: 0, observed: 0 }
|
||||||
|
races[r.race].enroll += r.stu_enroll || 0
|
||||||
|
races[r.race].observed += r.observed_arrests || 0
|
||||||
|
}
|
||||||
|
const merged = Object.values(races)
|
||||||
|
|
||||||
|
// defaultSelectedKeys logic on pooled groups
|
||||||
|
const withArrests = merged.filter((r) => r.observed > 0)
|
||||||
|
if (withArrests.length > 0) {
|
||||||
|
return withArrests.map((r) => ({ race: r.race, sex: null, enroll: r.enroll, observed: r.observed }))
|
||||||
|
}
|
||||||
|
return [...merged]
|
||||||
|
.sort((a, b) => b.enroll - a.enroll)
|
||||||
|
.slice(0, 2)
|
||||||
|
.map((r) => ({ race: r.race, sex: null, enroll: r.enroll, observed: r.observed }))
|
||||||
|
} else {
|
||||||
|
// Unpooled: select groups with ≥1 arrest or top-2 by enrollment
|
||||||
|
const withArrests = usable.filter((r) => (r.observed_arrests || 0) > 0)
|
||||||
|
if (withArrests.length > 0) return withArrests
|
||||||
|
|
||||||
|
return [...usable]
|
||||||
|
.sort((a, b) => (b.stu_enroll || 0) - (a.stu_enroll || 0))
|
||||||
|
.slice(0, 2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Compute max x-value candidate from selected cells. */
|
||||||
|
function computeMaxX(selectedCells, allCells, pooled) {
|
||||||
|
let maxAcRate = 0
|
||||||
|
let maxDrawEstimate = 0
|
||||||
|
|
||||||
|
for (const cell of selectedCells) {
|
||||||
|
const enroll = cell.enroll || 0
|
||||||
|
if (enroll <= 0) continue
|
||||||
|
const observed = cell.observed || 0
|
||||||
|
|
||||||
|
// AC upper bound rate (always a candidate in computeRateDomain)
|
||||||
|
const acUpper = acUpperBound(observed, enroll)
|
||||||
|
const acRate = (acUpper / enroll) * 1000
|
||||||
|
if (acRate > maxAcRate) maxAcRate = acRate
|
||||||
|
|
||||||
|
// Estimated draw quantile — find the matching row in allCells for model estimates
|
||||||
|
let countUpper = 0
|
||||||
|
let foundRow = null
|
||||||
|
if (!pooled && cell.sex) {
|
||||||
|
foundRow = allCells.find((r) => r.race === cell.race && r.sex === cell.sex)
|
||||||
|
} else if (pooled) {
|
||||||
|
// For pooled, find the row with max count_upper for this race across both sexes
|
||||||
|
const raceRows = allCells.filter((r) => r.race === cell.race && ['F', 'M'].includes(r.sex))
|
||||||
|
for (const rr of raceRows) {
|
||||||
|
if ((rr.count_upper || 0) > countUpper) countUpper = rr.count_upper || 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!pooled && foundRow) {
|
||||||
|
countUpper = foundRow.count_upper || 0
|
||||||
|
}
|
||||||
|
|
||||||
|
const enroll_ = cell.enroll || 1
|
||||||
|
let drawEst = null
|
||||||
|
if (countUpper > 0) {
|
||||||
|
drawEst = (countUpper / enroll_) * 1000
|
||||||
|
} else {
|
||||||
|
// Conservative proxy via AC upper bound
|
||||||
|
const ac = acUpperBound(observed, enroll_)
|
||||||
|
drawEst = (ac / enroll_) * 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
if (drawEst > maxDrawEstimate) maxDrawEstimate = drawEst
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.max(maxAcRate, maxDrawEstimate)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
console.log('=== Rate Domain Analysis ===\n')
|
||||||
|
|
||||||
|
// Step 1: Get all states from the API
|
||||||
|
const statesResp = await apiFetch('/states?limit=500')
|
||||||
|
const stateSet = new Set(statesResp.data.map((r) => r.state))
|
||||||
|
const states = [...stateSet].sort()
|
||||||
|
console.log(`Found ${states.length} states\n`)
|
||||||
|
|
||||||
|
// Step 2: For each state, page through all estimates and collect per-district data
|
||||||
|
const districtData = new Map() // leaid -> { state, name, cells: [] }
|
||||||
|
let totalRows = 0
|
||||||
|
|
||||||
|
for (const state of states) {
|
||||||
|
console.log(`Fetching ${state}...`)
|
||||||
|
const firstResp = await apiFetch(`/estimates?state=${state}&limit=${LIMIT}`)
|
||||||
|
const total = firstResp.meta.total
|
||||||
|
const nPages = Math.ceil(total / LIMIT)
|
||||||
|
|
||||||
|
let rows = [...firstResp.data]
|
||||||
|
for (let page = 1; page < nPages; page++) {
|
||||||
|
process.stderr.write(` ${state} page ${page + 1}/${nPages}\r`)
|
||||||
|
const resp = await apiFetch(`/estimates?state=${state}&limit=${LIMIT}&page=${page}`)
|
||||||
|
rows.push(...resp.data)
|
||||||
|
}
|
||||||
|
totalRows += rows.length
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const key = `${row.state}|${row.leaid}`
|
||||||
|
if (!districtData.has(key)) {
|
||||||
|
districtData.set(key, { state: row.state, leaid: row.leaid, name: row.lea_name, cells: [] })
|
||||||
|
}
|
||||||
|
districtData.get(key).cells.push(row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nTotal rows fetched: ${totalRows}`)
|
||||||
|
console.log(`Total districts: ${districtData.size}\n`)
|
||||||
|
|
||||||
|
// Step 3: For each district, compute the max x-value for both pooled and unpooled modes
|
||||||
|
const results = []
|
||||||
|
let nPooled = 0
|
||||||
|
let nUnpooled = 0
|
||||||
|
|
||||||
|
for (const [key, dist] of districtData) {
|
||||||
|
const totalArrests = dist.cells.reduce((sum, r) => sum + (r.observed_arrests || 0), 0)
|
||||||
|
const pooled = totalArrests < POOL_THRESHOLD
|
||||||
|
if (pooled) nPooled++
|
||||||
|
else nUnpooled++
|
||||||
|
|
||||||
|
let maxX
|
||||||
|
if (pooled) {
|
||||||
|
// Pooled mode: AC bounds computed per race (F+M merged). Note that for
|
||||||
|
// pooled groups, the app shows a note but still computes AC bounds.
|
||||||
|
const selected = determineSelectedCells(dist.cells, true)
|
||||||
|
maxX = computeMaxX(selected, dist.cells, true)
|
||||||
|
} else {
|
||||||
|
const selected = determineSelectedCells(dist.cells, false)
|
||||||
|
maxX = computeMaxX(selected, dist.cells, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
let totalEnroll = 0
|
||||||
|
for (const cell of dist.cells) {
|
||||||
|
if ((cell.stu_enroll || 0) > 0) totalEnroll += cell.stu_enroll
|
||||||
|
}
|
||||||
|
|
||||||
|
results.push({
|
||||||
|
key, state: dist.state, leaid: dist.leaid, name: dist.name,
|
||||||
|
totalEnroll, pooled, maxX: maxX * 1.15, // HEADROOM factor from rateDomain.js
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Districts with sex pooling (total arrests < ${POOL_THRESHOLD}): ${nPooled} (${(nPooled / results.length * 100).toFixed(1)}%)`)
|
||||||
|
console.log(`Districts without pooling: ${nUnpooled} (${(nUnpooled / results.length * 100).toFixed(1)}%)\n`)
|
||||||
|
|
||||||
|
// Step 4: Analyze distribution of max x-values
|
||||||
|
const sorted = results.sort((a, b) => a.maxX - b.maxX)
|
||||||
|
const n = sorted.length
|
||||||
|
|
||||||
|
console.log('=== Distribution of Maximum X-Values (per 1,000 students) ===\n')
|
||||||
|
|
||||||
|
// Summary statistics — maxX already includes HEADROOM(1.15) factor
|
||||||
|
const percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 99.9]
|
||||||
|
console.log('Percentiles of max x-value (includes HEADROOM=1.15):')
|
||||||
|
for (const p of percentiles) {
|
||||||
|
const idx = Math.floor((p / 100) * (n - 1))
|
||||||
|
console.log(` ${p.toFixed(1)}th: ${sorted[idx].maxX.toFixed(2)} per 1,000`)
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n--- Threshold analysis ---')
|
||||||
|
const thresholds = [30, 40, 50, 60, 70, 80, 90, 100]
|
||||||
|
for (const thresh of thresholds) {
|
||||||
|
const count = sorted.filter((r) => r.maxX > thresh).length
|
||||||
|
const pct = (count / n) * 100
|
||||||
|
console.log(` Exceeds ${thresh}: ${count} districts (${pct.toFixed(2)}%)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 5: Break down by pooling status
|
||||||
|
console.log('\n--- By pooling status ---')
|
||||||
|
const pooledResults = sorted.filter((r) => r.pooled)
|
||||||
|
const unpooledResults = sorted.filter((r) => !r.pooled)
|
||||||
|
for (const thresh of [30, 50, 100]) {
|
||||||
|
const pClipped = pooledResults.filter((r) => r.maxX > thresh).length
|
||||||
|
const uClipped = unpooledResults.filter((r) => r.maxX > thresh).length
|
||||||
|
console.log(` Cap=${thresh}: pooled ${pClipped}/${pooledResults.length} (${(pClipped / pooledResults.length * 100).toFixed(1)}%), ` +
|
||||||
|
`unpooled ${uClipped}/${unpooledResults.length} (${(uClipped / unpooledResults.length * 100).toFixed(1)}%)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 6: Show top districts by max x-value, with enrollment context
|
||||||
|
console.log('\n--- Top 25 districts by max x-value ---')
|
||||||
|
const top = sorted.slice(-25).reverse()
|
||||||
|
for (const r of top) {
|
||||||
|
const pooledStr = r.pooled ? ' [pooled]' : ''
|
||||||
|
const clippedAt30 = r.maxX > 30 ? ' *** CLIPPED at 30' : ''
|
||||||
|
console.log(` ${r.state} | LEAID ${r.leaid} | enroll=${r.totalEnroll.toLocaleString()}${pooledStr} | ` +
|
||||||
|
`max_x=${r.maxX.toFixed(2)}/1000` + clippedAt30)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 7: Show realistic-size districts (enrollment > 500) that are clipped
|
||||||
|
console.log('\n--- Clipped districts with enrollment > 500 ---')
|
||||||
|
const realClipped = sorted.filter((r) => r.maxX > 30 && r.totalEnroll >= 500).sort((a, b) => b.maxX - a.maxX).slice(0, 15)
|
||||||
|
for (const r of realClipped) {
|
||||||
|
const pooledStr = r.pooled ? ' [pooled]' : ''
|
||||||
|
console.log(` ${r.state} | ${r.name} | enroll=${r.totalEnroll.toLocaleString()}${pooledStr} | ` +
|
||||||
|
`max_x=${r.maxX.toFixed(2)}/1000`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 8: Non-clipped districts for context
|
||||||
|
const notClipped = sorted.filter((r) => r.maxX <= 30).sort((a, b) => a.maxX - b.maxX)
|
||||||
|
console.log(`\n--- Non-clipped districts (max_x ≤ 30): ${notClipped.length} (${(notClipped.length / n * 100).toFixed(2)}%) ---`)
|
||||||
|
if (notClipped.length > 0) {
|
||||||
|
const midIdx = Math.floor(notClipped.length / 2)
|
||||||
|
console.log(' Sample non-clipped districts:')
|
||||||
|
for (let i = Math.max(0, midIdx - 3); i < Math.min(notClipped.length, midIdx + 4); i++) {
|
||||||
|
const r = notClipped[i]
|
||||||
|
console.log(` ${r.state} | ${r.name} | enroll=${r.totalEnroll.toLocaleString()} | max_x=${r.maxX.toFixed(2)}/1000`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 9: State-level summary (focusing on clipped counts)
|
||||||
|
console.log('\n--- Top 15 states by % of districts clipped ---')
|
||||||
|
const byState = {}
|
||||||
|
for (const r of results) {
|
||||||
|
if (!byState[r.state]) byState[r.state] = []
|
||||||
|
byState[r.state].push(r.maxX)
|
||||||
|
}
|
||||||
|
const stateStats = Object.entries(byState).map(([state, vals]) => ({
|
||||||
|
state, n: vals.length, median: percentile(vals, 50), p95: percentile(vals, 95),
|
||||||
|
max: Math.max(...vals), clipped: vals.filter((v) => v > 30).length,
|
||||||
|
})).sort((a, b) => (b.clipped / b.n) - (a.clipped / a.n))
|
||||||
|
|
||||||
|
for (const s of stateStats.slice(0, 15)) {
|
||||||
|
console.log(` ${s.state}: n=${s.n}, median=${s.median.toFixed(1)}, p95=${s.p95.toFixed(1)}, ` +
|
||||||
|
`max=${s.max.toFixed(1)}, clipped>30: ${s.clipped} (${(s.clipped / s.n * 100).toFixed(1)}%)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 10: Recommendation analysis — what cap would minimize clipping while staying bounded?
|
||||||
|
console.log('\n=== RECOMMENDATION ANALYSIS ===')
|
||||||
|
const capOptions = [30, 40, 50, 60, 75, 100]
|
||||||
|
for (const cap of capOptions) {
|
||||||
|
const clipped = sorted.filter((r) => r.maxX > cap).length
|
||||||
|
console.log(` Cap=${cap}: ${clipped} districts clipped (${(clipped / n * 100).toFixed(2)}%)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 11: Key findings summary
|
||||||
|
console.log('\n--- Key Findings ---')
|
||||||
|
const pctExceed30 = (sorted.filter((r) => r.maxX > 30).length / n) * 100
|
||||||
|
const pctExceed50 = (sorted.filter((r) => r.maxX > 50).length / n) * 100
|
||||||
|
console.log(`- ${pctExceed30.toFixed(2)}% of districts have a max x-value exceeding the current cap of 30`)
|
||||||
|
console.log(`- ${pctExceed50.toFixed(2)}% exceed 50 per 1,000`)
|
||||||
|
const p99 = sorted[Math.floor(0.99 * (n - 1))].maxX
|
||||||
|
const max = sorted[n - 1].maxX
|
||||||
|
console.log(`- 99th percentile: ${p99.toFixed(2)} per 1,000`)
|
||||||
|
console.log(`- Maximum observed: ${max.toFixed(2)} per 1,000`)
|
||||||
|
|
||||||
|
// Analyze the nature of clipped districts — are they sparse or not?
|
||||||
|
const extreme = sorted.filter((r) => r.maxX > 30).sort((a, b) => a.totalEnroll - b.totalEnroll)
|
||||||
|
console.log(`\n--- Clipped district enrollment distribution ---`)
|
||||||
|
for (const p of [10, 25, 50, 75, 90]) {
|
||||||
|
const idx = Math.floor((p / 100) * (extreme.length - 1))
|
||||||
|
console.log(` ${p}th percentile enrollment: ${extreme[idx].totalEnroll.toLocaleString()}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// How many clipped districts have "normal" school sizes (>1000 students)?
|
||||||
|
const normalClipped = extreme.filter((r) => r.totalEnroll >= 1000).length
|
||||||
|
console.log(`\n- ${normalClipped} of ${extreme.length} clipped districts have enrollment ≥ 1,000 (${(normalClipped / extreme.length * 100).toFixed(1)}%)`)
|
||||||
|
|
||||||
|
// Analyze what's driving the extremes — AC bounds vs draw estimates
|
||||||
|
console.log('\n--- What drives extreme values? ---')
|
||||||
|
const acDriven = sorted.filter((r) => r.maxX > 30 && !r.pooled).length
|
||||||
|
console.log(`- Unpooled districts clipped: ${acDriven} (${(acDriven / n * 100).toFixed(2)}% of all)`)
|
||||||
|
|
||||||
|
function percentile(arr, p) {
|
||||||
|
const s = [...arr].sort((a, b) => a - b)
|
||||||
|
return s[Math.floor((p / 100) * (s.length - 1))]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(console.error)
|
||||||
+136
-53
@@ -11,24 +11,104 @@ import { MODEL_QUADRANT_LABEL } from '../utils/labels.js'
|
|||||||
* read as a pair.
|
* read as a pair.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Says which interval is actually on screen, and offers the better one when it
|
||||||
|
* hasn't been fetched. The distinction is not cosmetic: summing each group's
|
||||||
|
* own bounds answers "what if every group hit its extreme at once", which is a
|
||||||
|
* far wider claim than "how many arrests does the model think there were".
|
||||||
|
*/
|
||||||
|
function TotalIntervalNote({ totals, usingExact, onRequest }) {
|
||||||
|
const base = { fontSize: '0.75rem', margin: 'var(--space-1) 0 0' }
|
||||||
|
|
||||||
|
if (usingExact) {
|
||||||
|
return (
|
||||||
|
<p style={{ ...base, color: 'var(--cv-ink-3)' }}>
|
||||||
|
The modeled band is the 95% interval of the district <em>total</em>, taken from{' '}
|
||||||
|
{(totals?.byYear && Object.values(totals.byYear).find(Boolean)?.nDraws?.toLocaleString()) || '500'}{' '}
|
||||||
|
posterior predictive draws — summed across student groups within each draw, then
|
||||||
|
summarized across draws.
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totals?.status === 'loading') {
|
||||||
|
return <p style={{ ...base, color: 'var(--cv-ink-3)' }}>Computing the total from posterior draws…</p>
|
||||||
|
}
|
||||||
|
|
||||||
|
const mb = totals?.bytes ? (totals.bytes / 1048576).toFixed(1) : null
|
||||||
|
return (
|
||||||
|
<p style={{ ...base, color: 'var(--cv-ink-3)' }}>
|
||||||
|
The modeled band sums each student group’s own 95% bounds, which is wider than the
|
||||||
|
interval of the total — it is the case where every group lands at its extreme in the same
|
||||||
|
draw.{' '}
|
||||||
|
{totals?.status === 'error' ? (
|
||||||
|
<>The exact total could not be computed from the draws for this district.</>
|
||||||
|
) : (
|
||||||
|
onRequest && (
|
||||||
|
<button type="button" onClick={onRequest} style={linkButton}>
|
||||||
|
Compute the exact interval from the draws{mb ? ` (${mb} MB)` : ''}
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
)}
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const linkButton = {
|
||||||
|
background: 'none',
|
||||||
|
border: 'none',
|
||||||
|
padding: 0,
|
||||||
|
font: 'inherit',
|
||||||
|
color: 'var(--cv-navy-600)',
|
||||||
|
textDecoration: 'underline',
|
||||||
|
cursor: 'pointer',
|
||||||
|
}
|
||||||
|
|
||||||
const WAVE_LABELS = { '15-16': '2015–16', '17-18': '2017–18', '21-22': '2021–22' }
|
const WAVE_LABELS = { '15-16': '2015–16', '17-18': '2017–18', '21-22': '2021–22' }
|
||||||
const HEADROOM = 26 // px reserved at the top of the plot so a point's rate label has room to sit above it
|
const HEADROOM = 26 // px reserved at the top of the plot so a point's rate label has room to sit above it
|
||||||
|
|
||||||
export default function ArrestsOverTime({ data, modelId }) {
|
// Matches RateDensityPanel/GroupDifference. These SVGs are rendered at
|
||||||
|
// width="100%" inside the same card, so the viewBox width sets the scale factor
|
||||||
|
// for everything in it: at 360 this chart's 0.7rem text came out roughly twice
|
||||||
|
// the size of the other charts' 0.62rem, which is what made it look like a
|
||||||
|
// different chart from a different app.
|
||||||
|
const WIDTH = 760
|
||||||
|
const HEIGHT = 300
|
||||||
|
const MARGIN = { top: 20, right: 24, bottom: 52, left: 56 }
|
||||||
|
// Keeps the first and last wave off the plot edges. Without it the end markers
|
||||||
|
// sit flush against the y-axis and the right border, so half of each diamond's
|
||||||
|
// glyph is visually clipped and its label has nowhere to go.
|
||||||
|
const X_PAD = 64
|
||||||
|
|
||||||
|
export default function ArrestsOverTime({ data, modelId, totals, onRequestExactTotals }) {
|
||||||
|
// Prefer the interval computed from the draws (summed within each draw) over
|
||||||
|
// the sum of each group's own bounds, which is not the interval of the total
|
||||||
|
// and comes out systematically too wide.
|
||||||
|
const exact = totals?.status === 'ready' ? totals.byYear : null
|
||||||
|
const series = data.map((d) => {
|
||||||
|
const iv = exact?.[d.year]
|
||||||
|
return iv
|
||||||
|
? { ...d, modeledMedian: iv.median, modeledLower: iv.lower, modeledUpper: iv.upper, exact: true }
|
||||||
|
: { ...d, exact: false }
|
||||||
|
})
|
||||||
|
const usingExact = series.some((d) => d.exact)
|
||||||
|
|
||||||
const maxArrests = Math.max(
|
const maxArrests = Math.max(
|
||||||
...data.map((d) => Math.max(d.arrests, d.modeledUpper ?? 0)),
|
...series.map((d) => Math.max(d.arrests, d.modeledUpper ?? 0)),
|
||||||
1
|
1
|
||||||
)
|
)
|
||||||
const { ticks, niceMax } = niceTicks(maxArrests)
|
const { ticks, niceMax } = niceTicks(maxArrests)
|
||||||
const modelLabel = MODEL_QUADRANT_LABEL[modelId] || modelId
|
const modelLabel = MODEL_QUADRANT_LABEL[modelId] || modelId
|
||||||
|
|
||||||
const width = 360
|
const width = WIDTH
|
||||||
const height = 280
|
const height = HEIGHT
|
||||||
const margin = { top: 40, right: 40, bottom: 60, left: 65 }
|
const margin = MARGIN
|
||||||
const innerWidth = width - margin.left - margin.right
|
const innerWidth = width - margin.left - margin.right
|
||||||
const innerHeight = height - margin.top - margin.bottom
|
const innerHeight = height - margin.top - margin.bottom
|
||||||
|
|
||||||
const xScale = (i) => (i / Math.max(data.length - 1, 1)) * innerWidth
|
const plotLeft = margin.left + X_PAD
|
||||||
|
const plotRight = margin.left + innerWidth - X_PAD
|
||||||
|
const xScale = (i) => plotLeft + (i / Math.max(series.length - 1, 1)) * (plotRight - plotLeft)
|
||||||
const yScale = (val) => HEADROOM + (innerHeight - HEADROOM) * (1 - val / niceMax)
|
const yScale = (val) => HEADROOM + (innerHeight - HEADROOM) * (1 - val / niceMax)
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -42,86 +122,87 @@ export default function ArrestsOverTime({ data, modelId }) {
|
|||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<svg width="100%" viewBox={`0 0 ${width} ${height}`} style={{ maxWidth: '100%', marginTop: 'var(--space-1)' }}>
|
<svg width="100%" viewBox={`0 0 ${width} ${height}`}
|
||||||
<rect x={margin.left} y={margin.top} width={innerWidth} height={innerHeight}
|
style={{ maxWidth: '100%', minWidth: '320px', marginTop: 'var(--space-1)' }}
|
||||||
fill="var(--cv-paper-2)" rx={4} />
|
role="img" aria-label="Total arrests by CRDC wave, observed against the modeled total">
|
||||||
|
|
||||||
{ticks.map((val) => {
|
{ticks.map((val) => {
|
||||||
const y = margin.top + yScale(val)
|
const y = margin.top + yScale(val)
|
||||||
return (
|
return (
|
||||||
<g key={`y-${val}`}>
|
<g key={`y-${val}`}>
|
||||||
<line x1={margin.left} y1={y} x2={margin.left + innerWidth} y2={y}
|
<line x1={margin.left} y1={y} x2={margin.left + innerWidth} y2={y}
|
||||||
stroke="var(--cv-rule)" strokeWidth={1} />
|
stroke="var(--cv-rule)" strokeWidth={1} />
|
||||||
<text x={margin.left - 8} y={y + 4} textAnchor="end"
|
<text x={margin.left - 8} y={y + 3} textAnchor="end"
|
||||||
fontSize="0.7rem" fill="var(--cv-ink-3)">{val.toLocaleString()}</text>
|
fontSize="0.62rem" fill="var(--cv-ink-3)">{val.toLocaleString()}</text>
|
||||||
</g>
|
</g>
|
||||||
)
|
)
|
||||||
})}
|
})}
|
||||||
|
|
||||||
<text x={15} y={margin.top + innerHeight / 2} textAnchor="middle"
|
{/* Baseline, so the wave labels read as sitting on an axis */}
|
||||||
fontSize="0.7rem" fill="var(--cv-ink-3)" transform={`rotate(-90 15 ${margin.top + innerHeight / 2})`}>
|
<line x1={margin.left} y1={margin.top + yScale(0)} x2={margin.left + innerWidth}
|
||||||
Total arrests (count)
|
y2={margin.top + yScale(0)} stroke="var(--cv-rule-strong)" strokeWidth={1} />
|
||||||
|
|
||||||
|
<text x={14} y={margin.top + innerHeight / 2} textAnchor="middle"
|
||||||
|
fontSize="0.64rem" fill="var(--cv-ink-3)" transform={`rotate(-90 14 ${margin.top + innerHeight / 2})`}>
|
||||||
|
Total arrests
|
||||||
</text>
|
</text>
|
||||||
|
|
||||||
{data.map((d, i) => (
|
{series.map((d, i) => (
|
||||||
<text key={d.year} x={margin.left + xScale(i)} y={height - margin.bottom + 15}
|
<text key={d.year} x={xScale(i)} y={margin.top + yScale(0) + 18}
|
||||||
textAnchor="middle" fontSize="0.7rem" fill="var(--cv-ink)">
|
textAnchor="middle" fontSize="0.62rem" fill="var(--cv-ink-2)">
|
||||||
{WAVE_LABELS[d.year] || d.label}
|
{WAVE_LABELS[d.year] || d.label}
|
||||||
</text>
|
</text>
|
||||||
))}
|
))}
|
||||||
|
|
||||||
<text x={margin.left + innerWidth / 2} y={height - 5} textAnchor="middle"
|
<text x={margin.left + innerWidth / 2} y={height - 6} textAnchor="middle"
|
||||||
fontSize="0.7rem" fill="var(--cv-ink-3)">CRDC wave</text>
|
fontSize="0.64rem" fill="var(--cv-ink-3)">CRDC wave</text>
|
||||||
|
|
||||||
{/* Modeled point-range per wave — drawn first, directly under the observed marks it pairs with */}
|
{/* Modeled point-range per wave — drawn first, directly under the
|
||||||
{data.map((d, i) => {
|
observed marks it pairs with, and kept visually subordinate (thinner,
|
||||||
|
no halo) so the observed series reads as the primary line. */}
|
||||||
|
{series.map((d, i) => {
|
||||||
if (d.modeledMedian == null) return null
|
if (d.modeledMedian == null) return null
|
||||||
const cx = margin.left + xScale(i)
|
const cx = xScale(i)
|
||||||
const cyMedian = margin.top + yScale(d.modeledMedian)
|
const cyMedian = margin.top + yScale(d.modeledMedian)
|
||||||
const cyLower = margin.top + yScale(d.modeledLower ?? d.modeledMedian)
|
const cyLower = margin.top + yScale(d.modeledLower ?? d.modeledMedian)
|
||||||
const cyUpper = margin.top + yScale(d.modeledUpper ?? d.modeledMedian)
|
const cyUpper = margin.top + yScale(d.modeledUpper ?? d.modeledMedian)
|
||||||
return (
|
return (
|
||||||
<g key={`modeled-${d.year}`}>
|
<g key={`modeled-${d.year}`}>
|
||||||
<line x1={cx} y1={cyLower} x2={cx} y2={cyUpper} stroke={MODELED_AGGREGATE_COLOR} strokeWidth={2} />
|
<line x1={cx} y1={cyLower} x2={cx} y2={cyUpper}
|
||||||
<circle cx={cx} cy={cyMedian} r={4.5} fill={MODELED_AGGREGATE_COLOR} stroke="#fff" strokeWidth={1.25} />
|
stroke={MODELED_AGGREGATE_COLOR} strokeWidth={1.5} strokeLinecap="round" opacity={0.75} />
|
||||||
|
<circle cx={cx} cy={cyMedian} r={3.5} fill={MODELED_AGGREGATE_COLOR}
|
||||||
|
stroke="var(--cv-paper)" strokeWidth={1.25} />
|
||||||
</g>
|
</g>
|
||||||
)
|
)
|
||||||
})}
|
})}
|
||||||
|
|
||||||
{/* Observed line */}
|
{/* Observed line */}
|
||||||
{data.length > 1 && (
|
{series.length > 1 && (
|
||||||
<polyline
|
<polyline
|
||||||
points={data.map((d, i) => `${margin.left + xScale(i)},${margin.top + yScale(d.arrests)}`).join(' ')}
|
points={series.map((d, i) => `${xScale(i)},${margin.top + yScale(d.arrests)}`).join(' ')}
|
||||||
fill="none" stroke={OBSERVED_MARK_COLOR} strokeWidth={2.5}
|
fill="none" stroke={OBSERVED_MARK_COLOR} strokeWidth={2}
|
||||||
strokeLinejoin="round" strokeLinecap="round"
|
strokeLinejoin="round" strokeLinecap="round"
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Observed diamonds + rate-per-1k labels (halo behind the text keeps
|
{/* Observed diamonds + rate-per-1k labels. X_PAD keeps the end markers
|
||||||
it legible over the modeled whisker sharing the same column) */}
|
clear of the plot edges, so every label can be centred over its own
|
||||||
{data.map((d, i) => {
|
point instead of being anchored outward to avoid an overflow. */}
|
||||||
const cx = margin.left + xScale(i)
|
{series.map((d, i) => {
|
||||||
|
const cx = xScale(i)
|
||||||
const cy = margin.top + yScale(d.arrests)
|
const cy = margin.top + yScale(d.arrests)
|
||||||
const ratePerK = d.enroll > 0 ? (d.arrests / (d.enroll / 1000)).toFixed(2) : '0.0'
|
const ratePerK = d.enroll > 0 ? (d.arrests / (d.enroll / 1000)).toFixed(2) : '0.00'
|
||||||
// The first/last points sit flush on the plot's left/right edge, so a
|
|
||||||
// center-anchored label above them would overflow into the y-axis
|
|
||||||
// ticks or off the right edge — anchor those two outward instead.
|
|
||||||
const isFirst = i === 0
|
|
||||||
const isLast = i === data.length - 1
|
|
||||||
const anchor = isFirst ? 'start' : isLast ? 'end' : 'middle'
|
|
||||||
const labelX = isFirst ? cx + 7 : isLast ? cx - 7 : cx
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<g key={d.year}>
|
<g key={d.year}>
|
||||||
<rect x={cx - 5} y={cy - 5} width={10} height={10}
|
<rect x={cx - 4.5} y={cy - 4.5} width={9} height={9}
|
||||||
fill={OBSERVED_MARK_COLOR} stroke="#fff" strokeWidth={1.5}
|
fill={OBSERVED_MARK_COLOR} stroke="var(--cv-paper)" strokeWidth={1.5}
|
||||||
transform={`rotate(45 ${cx} ${cy})`} />
|
transform={`rotate(45 ${cx} ${cy})`} />
|
||||||
<text x={labelX} y={cy - 16} textAnchor={anchor} fontSize="0.7rem" fontWeight={600}
|
<text x={cx} y={cy - 13} textAnchor="middle" fontSize="0.66rem" fontWeight={700}
|
||||||
fill="var(--cv-ink)" stroke="var(--cv-paper-2)" strokeWidth={3}
|
fill="var(--cv-ink)" stroke="var(--cv-paper)" strokeWidth={3.5}
|
||||||
strokeLinejoin="round" paintOrder="stroke">{ratePerK}</text>
|
strokeLinejoin="round" paintOrder="stroke">
|
||||||
<text x={labelX} y={cy - 7} textAnchor={anchor} fontSize="0.6rem"
|
{ratePerK}
|
||||||
fill="var(--cv-ink-3)" stroke="var(--cv-paper-2)" strokeWidth={3}
|
<tspan fontSize="0.58rem" fontWeight={500} fill="var(--cv-ink-3)"> per 1,000</tspan>
|
||||||
strokeLinejoin="round" paintOrder="stroke">per 1k</text>
|
</text>
|
||||||
</g>
|
</g>
|
||||||
)
|
)
|
||||||
})}
|
})}
|
||||||
@@ -129,15 +210,17 @@ export default function ArrestsOverTime({ data, modelId }) {
|
|||||||
|
|
||||||
<ChartLegend items={[
|
<ChartLegend items={[
|
||||||
{ shape: 'diamond', color: OBSERVED_MARK_COLOR, label: 'Observed' },
|
{ shape: 'diamond', color: OBSERVED_MARK_COLOR, label: 'Observed' },
|
||||||
// 95%, not 90%: these bars are the API's count_lower/count_upper, and
|
// 95%, not 90%: the API's count_lower/count_upper are a 95% interval
|
||||||
// validate_interval() defaults to 95 (the app never passes interval=).
|
// (validate_interval() defaults to 95), and the draws-based total is
|
||||||
|
// computed at the same mass, so the two are directly comparable.
|
||||||
{ shape: 'dot', color: MODELED_AGGREGATE_COLOR, label: 'Modeled (median + 95% interval)' },
|
{ shape: 'dot', color: MODELED_AGGREGATE_COLOR, label: 'Modeled (median + 95% interval)' },
|
||||||
]} />
|
]} />
|
||||||
|
|
||||||
|
<TotalIntervalNote totals={totals} usingExact={usingExact} onRequest={onRequestExactTotals} />
|
||||||
|
|
||||||
<p style={{ fontSize: '0.75rem', color: 'var(--cv-ink-3)', marginTop: 'var(--space-1)' }}>
|
<p style={{ fontSize: '0.75rem', color: 'var(--cv-ink-3)', marginTop: 'var(--space-1)' }}>
|
||||||
Rate per 1,000 students labeled above each observed point. The modeled total sums each
|
Rate per 1,000 students labeled above each observed point. Data from CRDC waves 2015–16
|
||||||
selected group's median/interval independently, which approximates but is not exactly the
|
through 2021–22.
|
||||||
median of the combined total. Data from CRDC waves 2015–16 through 2021–22.
|
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { agrestiCoull } from '../utils/agrestiCoull.js'
|
|||||||
import { densityProfile } from '../utils/densityProfile.js'
|
import { densityProfile } from '../utils/densityProfile.js'
|
||||||
import { displayDraws } from '../utils/districtGroups.js'
|
import { displayDraws } from '../utils/districtGroups.js'
|
||||||
import { computeRateDomain } from '../utils/rateDomain.js'
|
import { computeRateDomain } from '../utils/rateDomain.js'
|
||||||
|
import { layoutPeakLabels } from '../utils/labelLayout.js'
|
||||||
import { toRates } from '../utils/pooling.js'
|
import { toRates } from '../utils/pooling.js'
|
||||||
import { densityCurve, fitSkewedInterval } from '../utils/distributionApprox.js'
|
import { densityCurve, fitSkewedInterval } from '../utils/distributionApprox.js'
|
||||||
import ApproxNote from '../components/ApproxNote.jsx'
|
import ApproxNote from '../components/ApproxNote.jsx'
|
||||||
@@ -39,6 +40,19 @@ const ROW_HEIGHT = 118
|
|||||||
const COMPARE_ROW_HEIGHT = 74
|
const COMPARE_ROW_HEIGHT = 74
|
||||||
const RAIL_HEIGHT = 26
|
const RAIL_HEIGHT = 26
|
||||||
|
|
||||||
|
// Direct labels live in a reserved band above each row rather than floating at
|
||||||
|
// each curve's apex. Every density is normalized to its own peak, so all the
|
||||||
|
// apexes sit at the same height — placing labels there stacks them on one line
|
||||||
|
// and they overprint into gibberish as soon as two groups have similar rates.
|
||||||
|
const LABEL_FONT_PX = 10.4 // 0.65rem
|
||||||
|
const COMPACT_LABEL_FONT_PX = 9.3 // 0.58rem
|
||||||
|
// Comfortably above the rendered line box (a 10.4px label measures ~12.1px tall
|
||||||
|
// with descenders), so adjacent lanes clear each other instead of just touching.
|
||||||
|
const LABEL_LINE_HEIGHT = 14
|
||||||
|
const COMPACT_LABEL_LINE_HEIGHT = 12
|
||||||
|
const LABEL_GAP = 8
|
||||||
|
const SPEC_LABEL_HEIGHT = 13
|
||||||
|
|
||||||
export default function RateDensityPanel({
|
export default function RateDensityPanel({
|
||||||
groups,
|
groups,
|
||||||
selectedKeys,
|
selectedKeys,
|
||||||
@@ -142,7 +156,11 @@ export default function RateDensityPanel({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{anyApproximated && <ApproxNote />}
|
{/* Only once the fetch has settled. While it is in flight every group is
|
||||||
|
nominally "approximated" (there are no draws yet), so showing the note
|
||||||
|
then would claim a fallback that hasn't happened — next to a spinner
|
||||||
|
saying the real draws are still coming. */}
|
||||||
|
{!loading && anyApproximated && <ApproxNote />}
|
||||||
|
|
||||||
{selected.length === 0 ? (
|
{selected.length === 0 ? (
|
||||||
<p style={emptyStyle}>
|
<p style={emptyStyle}>
|
||||||
@@ -207,6 +225,12 @@ function profileFor(group, counts, enroll, niceMax) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** x of a profile's tallest point, clamped into the plotted domain. */
|
||||||
|
function peakX(profile, x, niceMax) {
|
||||||
|
const peak = profile.points.reduce((best, p) => (p.y > best.y ? p : best), profile.points[0])
|
||||||
|
return x(Math.min(peak.x, niceMax))
|
||||||
|
}
|
||||||
|
|
||||||
function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, compareAll }) {
|
function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, compareAll }) {
|
||||||
const rows = activeModels.map((model) => ({
|
const rows = activeModels.map((model) => ({
|
||||||
model,
|
model,
|
||||||
@@ -217,11 +241,35 @@ function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, comp
|
|||||||
const hasAnything = rows.some((r) => r.entries.some((e) => e.profile))
|
const hasAnything = rows.some((r) => r.entries.some((e) => e.profile))
|
||||||
const rowHeight = compareAll ? COMPARE_ROW_HEIGHT : ROW_HEIGHT
|
const rowHeight = compareAll ? COMPARE_ROW_HEIGHT : ROW_HEIGHT
|
||||||
const innerWidth = CHART_WIDTH - MARGIN.left - MARGIN.right
|
const innerWidth = CHART_WIDTH - MARGIN.left - MARGIN.right
|
||||||
const bandHeight = rowHeight + RAIL_HEIGHT
|
const fontPx = compareAll ? COMPACT_LABEL_FONT_PX : LABEL_FONT_PX
|
||||||
const height = MARGIN.top + rows.length * bandHeight + MARGIN.bottom
|
const lineHeight = compareAll ? COMPACT_LABEL_LINE_HEIGHT : LABEL_LINE_HEIGHT
|
||||||
|
|
||||||
const x = scaleLinear().domain([0, domain.niceMax]).range([MARGIN.left, MARGIN.left + innerWidth])
|
const x = scaleLinear().domain([0, domain.niceMax]).range([MARGIN.left, MARGIN.left + innerWidth])
|
||||||
|
|
||||||
|
// Lay the labels out before sizing the SVG: how many lanes they need decides
|
||||||
|
// how much room each row has to reserve above its curves.
|
||||||
|
let cursor = MARGIN.top
|
||||||
|
const placedRows = rows.map((row) => {
|
||||||
|
const drawable = row.entries.filter((e) => e.profile)
|
||||||
|
const { labels, lanes } = layoutPeakLabels(
|
||||||
|
drawable.map((e) => ({
|
||||||
|
key: e.group.key,
|
||||||
|
x: peakX(e.profile, x, domain.niceMax),
|
||||||
|
text: e.group.shortLabel,
|
||||||
|
color: raceColor(e.group.race),
|
||||||
|
})),
|
||||||
|
{ min: MARGIN.left, max: MARGIN.left + innerWidth, fontPx, gap: LABEL_GAP },
|
||||||
|
)
|
||||||
|
const specLabel = compareAll ? SPEC_LABEL_HEIGHT : 0
|
||||||
|
const band = specLabel + (lanes > 0 ? lanes * lineHeight + 3 : 0)
|
||||||
|
const top = cursor
|
||||||
|
cursor += band + rowHeight + RAIL_HEIGHT
|
||||||
|
return { row, drawable, labels, band, specLabel, top, lineHeight, fontPx }
|
||||||
|
})
|
||||||
|
|
||||||
|
const plotBottom = cursor
|
||||||
|
const height = plotBottom + MARGIN.bottom
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<figure style={{ margin: '0 0 var(--space-2)' }}>
|
<figure style={{ margin: '0 0 var(--space-2)' }}>
|
||||||
{label && <figcaption style={panelLabelStyle}>{label}</figcaption>}
|
{label && <figcaption style={panelLabelStyle}>{label}</figcaption>}
|
||||||
@@ -242,19 +290,18 @@ function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, comp
|
|||||||
x1={x(t)}
|
x1={x(t)}
|
||||||
y1={MARGIN.top}
|
y1={MARGIN.top}
|
||||||
x2={x(t)}
|
x2={x(t)}
|
||||||
y2={MARGIN.top + rows.length * bandHeight}
|
y2={plotBottom}
|
||||||
stroke="var(--cv-rule)"
|
stroke="var(--cv-rule)"
|
||||||
strokeWidth={1}
|
strokeWidth={1}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
|
|
||||||
{rows.map((row, i) => (
|
{placedRows.map((placed) => (
|
||||||
<ModelRow
|
<ModelRow
|
||||||
key={row.model}
|
key={placed.row.model}
|
||||||
row={row}
|
placed={placed}
|
||||||
compareAll={compareAll}
|
compareAll={compareAll}
|
||||||
x={x}
|
x={x}
|
||||||
top={MARGIN.top + i * bandHeight}
|
|
||||||
rowHeight={rowHeight}
|
rowHeight={rowHeight}
|
||||||
acByKey={acByKey}
|
acByKey={acByKey}
|
||||||
niceMax={domain.niceMax}
|
niceMax={domain.niceMax}
|
||||||
@@ -265,7 +312,7 @@ function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, comp
|
|||||||
<text
|
<text
|
||||||
key={t}
|
key={t}
|
||||||
x={x(t)}
|
x={x(t)}
|
||||||
y={MARGIN.top + rows.length * bandHeight + 15}
|
y={plotBottom + 15}
|
||||||
textAnchor="middle"
|
textAnchor="middle"
|
||||||
fontSize="0.62rem"
|
fontSize="0.62rem"
|
||||||
fill="var(--cv-ink-3)"
|
fill="var(--cv-ink-3)"
|
||||||
@@ -289,10 +336,10 @@ function SexPanel({ label, sex, domain, rowsByModel, activeModels, acByKey, comp
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function ModelRow({ row, compareAll, x, top, rowHeight, acByKey, niceMax }) {
|
function ModelRow({ placed, compareAll, x, rowHeight, acByKey, niceMax }) {
|
||||||
const baselineY = top + rowHeight
|
const { row, drawable, labels, band, specLabel, top, lineHeight, fontPx } = placed
|
||||||
|
const baselineY = top + band + rowHeight
|
||||||
const peakHeight = rowHeight * 0.86
|
const peakHeight = rowHeight * 0.86
|
||||||
const drawable = row.entries.filter((e) => e.profile)
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<g>
|
<g>
|
||||||
@@ -302,7 +349,28 @@ function ModelRow({ row, compareAll, x, top, rowHeight, acByKey, niceMax }) {
|
|||||||
</text>
|
</text>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{drawable.map((entry, i) => (
|
{/* Direct labels, packed into lanes so overlapping densities stay legible.
|
||||||
|
Colour is what ties each one to its curve — the same hue the table's
|
||||||
|
swatch uses — so no leader lines are needed. */}
|
||||||
|
{labels.map((label) => (
|
||||||
|
<text
|
||||||
|
key={label.key}
|
||||||
|
x={label.x}
|
||||||
|
y={top + specLabel + label.lane * lineHeight + fontPx}
|
||||||
|
textAnchor={label.anchor}
|
||||||
|
fontSize={`${fontPx}px`}
|
||||||
|
fontWeight={700}
|
||||||
|
fill={label.color}
|
||||||
|
stroke="var(--cv-paper)"
|
||||||
|
strokeWidth={3}
|
||||||
|
strokeLinejoin="round"
|
||||||
|
paintOrder="stroke"
|
||||||
|
>
|
||||||
|
{label.text}
|
||||||
|
</text>
|
||||||
|
))}
|
||||||
|
|
||||||
|
{drawable.map((entry) => (
|
||||||
<GroupArea
|
<GroupArea
|
||||||
key={entry.group.key}
|
key={entry.group.key}
|
||||||
entry={entry}
|
entry={entry}
|
||||||
@@ -314,8 +382,6 @@ function ModelRow({ row, compareAll, x, top, rowHeight, acByKey, niceMax }) {
|
|||||||
// what the reader is comparing here is location and spread, not peak
|
// what the reader is comparing here is location and spread, not peak
|
||||||
// height.
|
// height.
|
||||||
peakHeight={peakHeight}
|
peakHeight={peakHeight}
|
||||||
labelIndex={i}
|
|
||||||
compact={compareAll}
|
|
||||||
niceMax={niceMax}
|
niceMax={niceMax}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
@@ -347,7 +413,7 @@ function ModelRow({ row, compareAll, x, top, rowHeight, acByKey, niceMax }) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function GroupArea({ entry, x, baselineY, peakHeight, labelIndex, compact, niceMax }) {
|
function GroupArea({ entry, x, baselineY, peakHeight, niceMax }) {
|
||||||
const { profile, group } = entry
|
const { profile, group } = entry
|
||||||
const color = raceColor(group.race)
|
const color = raceColor(group.race)
|
||||||
const y = scaleLinear().domain([0, profile.maxY || 1]).range([baselineY, baselineY - peakHeight])
|
const y = scaleLinear().domain([0, profile.maxY || 1]).range([baselineY, baselineY - peakHeight])
|
||||||
@@ -366,10 +432,6 @@ function GroupArea({ entry, x, baselineY, peakHeight, labelIndex, compact, niceM
|
|||||||
// still visually rhyming with the filled areas beside it.
|
// still visually rhyming with the filled areas beside it.
|
||||||
.curve(profile.kind === 'mass' ? curveStep : curveMonotoneX)
|
.curve(profile.kind === 'mass' ? curveStep : curveMonotoneX)
|
||||||
|
|
||||||
const peak = profile.points.reduce((best, p) => (p.y > best.y ? p : best), profile.points[0])
|
|
||||||
const peakX = x(Math.min(peak.x, niceMax))
|
|
||||||
const labelY = Math.max(baselineY - peakHeight - 2, y(peak.y) - 4 - (labelIndex % 2) * 11)
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<g>
|
<g>
|
||||||
<path d={areaGen(points)} fill={color} opacity={FILL_OPACITY} />
|
<path d={areaGen(points)} fill={color} opacity={FILL_OPACITY} />
|
||||||
@@ -380,20 +442,6 @@ function GroupArea({ entry, x, baselineY, peakHeight, labelIndex, compact, niceM
|
|||||||
strokeWidth={STROKE_WIDTH}
|
strokeWidth={STROKE_WIDTH}
|
||||||
strokeLinejoin="round"
|
strokeLinejoin="round"
|
||||||
/>
|
/>
|
||||||
{/* Direct-labelled at the peak so there is no legend to hunt through. */}
|
|
||||||
<text
|
|
||||||
x={peakX}
|
|
||||||
y={labelY}
|
|
||||||
textAnchor={peakX > x(niceMax) * 0.8 ? 'end' : 'middle'}
|
|
||||||
fontSize={compact ? '0.58rem' : '0.65rem'}
|
|
||||||
fontWeight={700}
|
|
||||||
fill={color}
|
|
||||||
stroke="var(--cv-paper)"
|
|
||||||
strokeWidth={3}
|
|
||||||
paintOrder="stroke"
|
|
||||||
>
|
|
||||||
{group.shortLabel}
|
|
||||||
</text>
|
|
||||||
</g>
|
</g>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||||
import * as api from '../hooks/useApi.js'
|
import * as api from '../hooks/useApi.js'
|
||||||
import { MODEL_QUADRANTS } from '../hooks/useApi.js'
|
import { MODEL_QUADRANTS } from '../hooks/useApi.js'
|
||||||
import { useDrawDistribution } from '../hooks/useDrawDistribution.js'
|
import { useDistrictTotalDraws, useDrawDistribution } from '../hooks/useDrawDistribution.js'
|
||||||
import {
|
import {
|
||||||
buildDisplayGroups,
|
buildDisplayGroups,
|
||||||
defaultDiffPair,
|
defaultDiffPair,
|
||||||
@@ -20,6 +20,11 @@ const DEFAULT_SPEC = 'unified_m4_mod' // three-year + referral rate
|
|||||||
const CURRENT_WAVE = '21-22'
|
const CURRENT_WAVE = '21-22'
|
||||||
const QUADRANT_MODELS = MODEL_QUADRANTS.map((q) => q.model)
|
const QUADRANT_MODELS = MODEL_QUADRANTS.map((q) => q.model)
|
||||||
|
|
||||||
|
// Below this, the draws for all three waves are fetched without asking. Sized
|
||||||
|
// from the real shards: Nevada's three waves are 0.31MB, California's 13.9MB
|
||||||
|
// and Texas's 14.3MB, so 3MB cleanly separates "free" from "worth a click".
|
||||||
|
const AUTO_TOTAL_DRAW_BYTES = 3 * 1024 * 1024
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The results page: what was observed (summary table), what the model says the
|
* The results page: what was observed (summary table), what the model says the
|
||||||
* rate is (density panel), and what it says about the gap between two groups
|
* rate is (density panel), and what it says about the gap between two groups
|
||||||
@@ -43,6 +48,7 @@ export default function ChartPanel({ district, state }) {
|
|||||||
// made in one mode must not be reapplied in the other.
|
// made in one mode must not be reapplied in the other.
|
||||||
const [selection, setSelection] = useState(null)
|
const [selection, setSelection] = useState(null)
|
||||||
const [pairSelection, setPairSelection] = useState(null)
|
const [pairSelection, setPairSelection] = useState(null)
|
||||||
|
const [exactTotals, setExactTotals] = useState(false)
|
||||||
|
|
||||||
// ——— Fetch: the three waves for the time series ———
|
// ——— Fetch: the three waves for the time series ———
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -54,6 +60,7 @@ export default function ChartPanel({ district, state }) {
|
|||||||
setSelection(null)
|
setSelection(null)
|
||||||
setPairSelection(null)
|
setPairSelection(null)
|
||||||
setCompareAll(false)
|
setCompareAll(false)
|
||||||
|
setExactTotals(false)
|
||||||
|
|
||||||
async function run() {
|
async function run() {
|
||||||
const waves = {}
|
const waves = {}
|
||||||
@@ -154,6 +161,22 @@ export default function ChartPanel({ district, state }) {
|
|||||||
year: CURRENT_WAVE,
|
year: CURRENT_WAVE,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// The time series' band, computed from the draws rather than by summing each
|
||||||
|
// group's own bounds. Fetched automatically only where it is cheap — three
|
||||||
|
// waves is 0.3MB for Nevada but 13.9MB for California — otherwise the chart
|
||||||
|
// offers it as an explicit choice with the size shown.
|
||||||
|
const totals = useDistrictTotalDraws({
|
||||||
|
leaid: district.leaid,
|
||||||
|
state,
|
||||||
|
model: WAVE_MODEL,
|
||||||
|
years: ALL_WAVES,
|
||||||
|
enabled: exactTotals,
|
||||||
|
})
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (totals.bytes > 0 && totals.bytes <= AUTO_TOTAL_DRAW_BYTES) setExactTotals(true)
|
||||||
|
}, [totals.bytes])
|
||||||
|
|
||||||
if (loading || !waveData) return <LoadingCharts />
|
if (loading || !waveData) return <LoadingCharts />
|
||||||
|
|
||||||
const timeSeriesData = ALL_WAVES.map((year) => {
|
const timeSeriesData = ALL_WAVES.map((year) => {
|
||||||
@@ -223,7 +246,12 @@ export default function ChartPanel({ district, state }) {
|
|||||||
onPairChange={handlePairChange}
|
onPairChange={handlePairChange}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<ArrestsOverTime data={timeSeriesData} modelId={WAVE_MODEL} />
|
<ArrestsOverTime
|
||||||
|
data={timeSeriesData}
|
||||||
|
modelId={WAVE_MODEL}
|
||||||
|
totals={totals}
|
||||||
|
onRequestExactTotals={() => setExactTotals(true)}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div
|
<div
|
||||||
@@ -241,6 +269,20 @@ export default function ChartPanel({ district, state }) {
|
|||||||
without referral-rate covariates. All rates are per 1,000 students. Modelled distributions
|
without referral-rate covariates. All rates are per 1,000 students. Modelled distributions
|
||||||
are drawn from the published posterior predictive draws, fetched in the browser.
|
are drawn from the published posterior predictive draws, fetched in the browser.
|
||||||
</p>
|
</p>
|
||||||
|
<p style={{ fontSize: '0.85rem', color: 'var(--cv-ink-2)', marginTop: 'var(--space-2)' }}>
|
||||||
|
<strong>Student groups not shown.</strong> The CRDC also reports arrests for Asian,
|
||||||
|
Hawaiian/Pacific Islander, and multiracial students. The underlying research does not
|
||||||
|
model these groups due to extreme sparsity in the outcome of interest and computational
|
||||||
|
limits. Fitting every reported group would also have meant fitting 70
|
||||||
|
stratified models rather than 40. Their absence here reflects that scope, not a finding
|
||||||
|
about those students, and no estimate for them should be inferred from this page. See the{' '}
|
||||||
|
<a href="https://crdc-api.civilytics.org/api/v1/" target="_blank" rel="noopener noreferrer">
|
||||||
|
white paper
|
||||||
|
</a>{' '}
|
||||||
|
for the full rationale. Arrests that cannot be disaggregated by both race and sex —
|
||||||
|
including Section 504 students — are excluded upstream, as are the small number of arrests in
|
||||||
|
schools not enrolling grade 7 or above.
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { useEffect, useMemo, useState } from 'react'
|
import { useEffect, useMemo, useState } from 'react'
|
||||||
import { getDb } from '../utils/duckdbClient.js'
|
import { getDb } from '../utils/duckdbClient.js'
|
||||||
import { groupKey, isCompleteDrawSet } from '../utils/drawGroups.js'
|
import { groupKey, isCompleteDrawSet } from '../utils/drawGroups.js'
|
||||||
|
import { totalInterval, totalPerDraw } from '../utils/districtTotal.js'
|
||||||
|
|
||||||
const HF_DATASET = 'civilytics/crdc-school-arrest-rates'
|
const HF_DATASET = 'civilytics/crdc-school-arrest-rates'
|
||||||
const HF_BASE = `https://huggingface.co/datasets/${HF_DATASET}/resolve/main/parquet`
|
const HF_BASE = `https://huggingface.co/datasets/${HF_DATASET}/resolve/main/parquet`
|
||||||
@@ -24,6 +25,57 @@ function shardDir(model, year, state) {
|
|||||||
return `model_id=${model}/YEAR=${year}/LEA_STATE=${state}`
|
return `model_id=${model}/YEAR=${year}/LEA_STATE=${state}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Directory listings are ~1KB and carry each part's byte size, so the UI can
|
||||||
|
// tell the reader what a fetch will cost before committing to it. Cached
|
||||||
|
// separately from the buffers: listing a shard is cheap, downloading it is not.
|
||||||
|
const listingCache = new Map()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {Promise<Array<{part: number, size: number}>>} parquet parts, in order
|
||||||
|
*/
|
||||||
|
function listShardEntries(model, year, state) {
|
||||||
|
const dir = shardDir(model, year, state)
|
||||||
|
if (!listingCache.has(dir)) {
|
||||||
|
listingCache.set(
|
||||||
|
dir,
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${HF_TREE}/${dir}`)
|
||||||
|
if (!res.ok) throw new Error(`tree listing HTTP ${res.status}`)
|
||||||
|
const entries = await res.json()
|
||||||
|
return entries
|
||||||
|
.filter((e) => e?.type === 'file' && /\/data_\d+\.parquet$/.test(e.path || ''))
|
||||||
|
.map((e) => ({ part: Number(e.path.match(/data_(\d+)\.parquet$/)[1]), size: e.size || 0 }))
|
||||||
|
.sort((a, b) => a.part - b.part)
|
||||||
|
} catch (err) {
|
||||||
|
listingCache.delete(dir)
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
})(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return listingCache.get(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Total bytes of the draw shards for one model across several years — what a
|
||||||
|
* "compute this from the draws" action will actually download.
|
||||||
|
*
|
||||||
|
* @returns {Promise<number>} bytes, or 0 if the size can't be determined
|
||||||
|
*/
|
||||||
|
export async function measureShardBytes(model, years, state) {
|
||||||
|
try {
|
||||||
|
const sizes = await Promise.all(
|
||||||
|
years.map(async (year) =>
|
||||||
|
(await listShardEntries(model, year, state)).reduce((sum, p) => sum + p.size, 0),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return sizes.reduce((sum, n) => sum + n, 0)
|
||||||
|
} catch {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Lists the parquet parts published for one (model, year, state).
|
* Lists the parquet parts published for one (model, year, state).
|
||||||
*
|
*
|
||||||
@@ -42,13 +94,7 @@ function shardDir(model, year, state) {
|
|||||||
async function listShardParts(model, year, state) {
|
async function listShardParts(model, year, state) {
|
||||||
const dir = shardDir(model, year, state)
|
const dir = shardDir(model, year, state)
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`${HF_TREE}/${dir}`)
|
const parts = await listShardEntries(model, year, state)
|
||||||
if (!res.ok) throw new Error(`tree listing HTTP ${res.status}`)
|
|
||||||
const entries = await res.json()
|
|
||||||
const parts = entries
|
|
||||||
.filter((e) => e?.type === 'file' && /\/data_\d+\.parquet$/.test(e.path || ''))
|
|
||||||
.map((e) => ({ part: Number(e.path.match(/data_(\d+)\.parquet$/)[1]), path: e.path }))
|
|
||||||
.sort((a, b) => a.part - b.part)
|
|
||||||
if (parts.length > 0) return parts.map((p) => p.part)
|
if (parts.length > 0) return parts.map((p) => p.part)
|
||||||
throw new Error('tree listing contained no parquet parts')
|
throw new Error('tree listing contained no parquet parts')
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -139,12 +185,18 @@ async function fetchModelCounts(db, { leaid, state, model, year }) {
|
|||||||
// a short draw set: its density and interval would be computed off a
|
// a short draw set: its density and interval would be computed off a
|
||||||
// biased subsample and look identical on screen to a complete one.
|
// biased subsample and look identical on screen to a complete one.
|
||||||
const counts = {}
|
const counts = {}
|
||||||
|
let dropped = 0
|
||||||
for (const [key, arr] of Object.entries(raw)) {
|
for (const [key, arr] of Object.entries(raw)) {
|
||||||
if (isCompleteDrawSet(arr, nDraws)) counts[key] = arr
|
if (isCompleteDrawSet(arr, nDraws)) counts[key] = arr
|
||||||
else console.warn('useDrawDistribution: dropping incomplete draw set for', { leaid, model, key, got: arr.length, want: nDraws })
|
else {
|
||||||
|
dropped += 1
|
||||||
|
console.warn('useDrawDistribution: dropping incomplete draw set for', { leaid, model, key, got: arr.length, want: nDraws })
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return Object.keys(counts).length > 0 ? { counts, nDraws } : null
|
// `dropped` matters to any consumer that aggregates *across* groups (the
|
||||||
|
// district total): a missing group there is an undercount, not a gap.
|
||||||
|
return Object.keys(counts).length > 0 ? { counts, nDraws, dropped } : null
|
||||||
} finally {
|
} finally {
|
||||||
if (conn) await conn.close()
|
if (conn) await conn.close()
|
||||||
}
|
}
|
||||||
@@ -257,3 +309,89 @@ export function useDrawDistribution({ leaid, state, models, year }) {
|
|||||||
|
|
||||||
return { status, byModel, nDraws }
|
return { status, byModel, nDraws }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The district's total arrest count per wave, with a 95% interval computed from
|
||||||
|
* the draws — summed within each draw, then summarized across draws.
|
||||||
|
*
|
||||||
|
* Gated behind `enabled` because the cost is wildly uneven: three waves of
|
||||||
|
* Nevada is 0.3MB, but California is 13.9MB and Texas 14.3MB. `bytes` is
|
||||||
|
* probed from the directory listings up front (~1KB per year) so the caller can
|
||||||
|
* either fetch automatically when it's cheap or show the reader the price
|
||||||
|
* first.
|
||||||
|
*
|
||||||
|
* @param {{leaid: string, state: string, model: string, years: string[], enabled: boolean}} params
|
||||||
|
* @returns {{status: 'idle'|'loading'|'ready'|'error',
|
||||||
|
* byYear: Record<string, {lower:number, median:number, upper:number, nDraws:number}> | null,
|
||||||
|
* bytes: number}}
|
||||||
|
*/
|
||||||
|
export function useDistrictTotalDraws({ leaid, state, model, years, enabled }) {
|
||||||
|
const [status, setStatus] = useState('idle')
|
||||||
|
const [byYear, setByYear] = useState(null)
|
||||||
|
const [bytes, setBytes] = useState(0)
|
||||||
|
|
||||||
|
const yearsSignature = (years || []).join(',')
|
||||||
|
|
||||||
|
// Probe sizes regardless of `enabled` — this is what lets the UI decide.
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false
|
||||||
|
setBytes(0)
|
||||||
|
if (!state || !model || !yearsSignature) return
|
||||||
|
measureShardBytes(model, yearsSignature.split(','), state).then((n) => {
|
||||||
|
if (!cancelled) setBytes(n)
|
||||||
|
})
|
||||||
|
return () => {
|
||||||
|
cancelled = true
|
||||||
|
}
|
||||||
|
}, [state, model, yearsSignature])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false
|
||||||
|
setStatus(enabled ? 'loading' : 'idle')
|
||||||
|
setByYear(null)
|
||||||
|
|
||||||
|
if (!enabled || !leaid || !state || !model || !yearsSignature) return
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
try {
|
||||||
|
const db = await getDb()
|
||||||
|
const yearList = yearsSignature.split(',')
|
||||||
|
const results = await Promise.all(
|
||||||
|
yearList.map(async (year) => {
|
||||||
|
try {
|
||||||
|
const model_ = await fetchModelCounts(db, { leaid, state, model, year })
|
||||||
|
if (!model_ || model_.dropped > 0) return [year, null]
|
||||||
|
const interval = totalInterval(totalPerDraw(model_.counts, model_.nDraws))
|
||||||
|
return [year, interval]
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`useDistrictTotalDraws: ${year} failed:`, err)
|
||||||
|
return [year, null]
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if (cancelled) return
|
||||||
|
const next = Object.fromEntries(results)
|
||||||
|
if (Object.values(next).some((v) => v !== null)) {
|
||||||
|
setByYear(next)
|
||||||
|
setStatus('ready')
|
||||||
|
} else {
|
||||||
|
setByYear(null)
|
||||||
|
setStatus('error')
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('useDistrictTotalDraws failed:', err)
|
||||||
|
if (!cancelled) {
|
||||||
|
setByYear(null)
|
||||||
|
setStatus('error')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
run()
|
||||||
|
return () => {
|
||||||
|
cancelled = true
|
||||||
|
}
|
||||||
|
}, [leaid, state, model, yearsSignature, enabled])
|
||||||
|
|
||||||
|
return { status, byYear, bytes }
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
/**
|
||||||
|
* The district-wide arrest total, taken from the posterior predictive draws.
|
||||||
|
*
|
||||||
|
* Why this exists: the time-series chart used to draw its band by summing each
|
||||||
|
* student group's own `count_lower`/`count_upper`. That is not the interval of
|
||||||
|
* the total. The sum of per-group 97.5th percentiles is the value you would see
|
||||||
|
* if *every* group simultaneously landed at its own extreme in the same draw,
|
||||||
|
* which is far less likely than any one group doing so — so the band came out
|
||||||
|
* systematically too wide. Summing within each draw and taking quantiles of the
|
||||||
|
* resulting totals answers the actual question: how many arrests does the model
|
||||||
|
* think this district had?
|
||||||
|
*
|
||||||
|
* This is `build_state_summary()`'s method from crdc-arrests/R/summarize_draws.R
|
||||||
|
* applied on a different axis — sum inside the draw, summarize across draws —
|
||||||
|
* the same operation `poolBySex` performs for sex pooling.
|
||||||
|
*
|
||||||
|
* One caveat to carry into any caption. Upstream, `draw_id` is renumbered per
|
||||||
|
* write batch, so draw *k* of one group is not the same posterior sample as
|
||||||
|
* draw *k* of another; measured cross-group correlation is ≈ 0.02. Summing at a
|
||||||
|
* draw index therefore convolves what are effectively independent draws. That
|
||||||
|
* is a reasonable model here — these are *posterior predictive* draws, whose
|
||||||
|
* observation noise is independent across groups by construction and dominates
|
||||||
|
* the parameter-level covariance — but it does mean the result should be read
|
||||||
|
* as a predictive total, not as a contrast that preserves parameter
|
||||||
|
* correlation. It is still much closer to the truth than summing bounds.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { quantile } from './kde.js'
|
||||||
|
import { isCompleteDrawSet } from './drawGroups.js'
|
||||||
|
|
||||||
|
/** Matches the API's default interval, so the two are directly comparable. */
|
||||||
|
export const TOTAL_INTERVAL_MASS = 0.95
|
||||||
|
|
||||||
|
/**
|
||||||
|
* District total at each draw index: the sum across every student group.
|
||||||
|
*
|
||||||
|
* Returns null unless *every* group has a complete draw set. A group silently
|
||||||
|
* missing from the sum would undercount the total at every draw and shift the
|
||||||
|
* whole interval down, which is indistinguishable on screen from a real result.
|
||||||
|
*
|
||||||
|
* @param {Record<string, number[]> | null | undefined} countsByGroup
|
||||||
|
* @param {number} nDraws
|
||||||
|
* @returns {number[] | null}
|
||||||
|
*/
|
||||||
|
export function totalPerDraw(countsByGroup, nDraws) {
|
||||||
|
const groups = Object.values(countsByGroup || {})
|
||||||
|
if (!groups.length || !(nDraws > 0)) return null
|
||||||
|
if (!groups.every((counts) => isCompleteDrawSet(counts, nDraws))) return null
|
||||||
|
|
||||||
|
const totals = new Array(nDraws).fill(0)
|
||||||
|
for (const counts of groups) {
|
||||||
|
for (let i = 0; i < nDraws; i++) totals[i] += counts[i]
|
||||||
|
}
|
||||||
|
return totals
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Narrowest interval containing `mass` of the values — the highest-density
|
||||||
|
* interval, matching what the API stores.
|
||||||
|
*
|
||||||
|
* The API's `count_lower`/`count_upper` are HPD bounds (`hpd_bounds_sql` in
|
||||||
|
* crdc-arrests/R/summarize_draws.R), not equal-tailed quantiles, and the white
|
||||||
|
* paper's figures use the same. Computing the total's interval the same way
|
||||||
|
* keeps one definition of "95% interval" on the page: the difference is only
|
||||||
|
* two or three arrests on a Clark County band of ~50, but the chart's fallback
|
||||||
|
* mode shows summed HPD bounds, and mixing conventions between the two modes
|
||||||
|
* would be a distinction with no explanation.
|
||||||
|
*
|
||||||
|
* Contiguous by construction, as the R original is. For a strongly bimodal
|
||||||
|
* posterior that is a simplification, but a count total summed across eight
|
||||||
|
* groups is unimodal in practice.
|
||||||
|
*
|
||||||
|
* @param {number[]} values
|
||||||
|
* @param {number} mass - e.g. 0.95
|
||||||
|
* @returns {[number, number]}
|
||||||
|
*/
|
||||||
|
export function hpdBounds(values, mass) {
|
||||||
|
const sorted = [...values].sort((a, b) => a - b)
|
||||||
|
const n = sorted.length
|
||||||
|
const span = Math.ceil(mass * n) - 1
|
||||||
|
if (span <= 0) return [sorted[0], sorted[0]]
|
||||||
|
if (span >= n - 1) return [sorted[0], sorted[n - 1]]
|
||||||
|
|
||||||
|
let best = [sorted[0], sorted[span]]
|
||||||
|
let bestWidth = sorted[span] - sorted[0]
|
||||||
|
for (let i = 1; i + span < n; i++) {
|
||||||
|
const width = sorted[i + span] - sorted[i]
|
||||||
|
if (width < bestWidth) {
|
||||||
|
bestWidth = width
|
||||||
|
best = [sorted[i], sorted[i + span]]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {number[] | null | undefined} totals - district total per draw
|
||||||
|
* @returns {{lower: number, median: number, upper: number, nDraws: number} | null}
|
||||||
|
*/
|
||||||
|
export function totalInterval(totals) {
|
||||||
|
if (!totals?.length) return null
|
||||||
|
const [lower, upper] = hpdBounds(totals, TOTAL_INTERVAL_MASS)
|
||||||
|
return { lower, median: quantile(totals, 0.5), upper, nDraws: totals.length }
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import { test } from 'node:test'
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import { TOTAL_INTERVAL_MASS, hpdBounds, totalInterval, totalPerDraw } from './districtTotal.js'
|
||||||
|
|
||||||
|
// ——— totalPerDraw ———
|
||||||
|
|
||||||
|
test('totalPerDraw: sums every group at the same draw index', () => {
|
||||||
|
const counts = { BL_F: [1, 2, 3], BL_M: [10, 20, 30], WH_F: [100, 200, 300] }
|
||||||
|
assert.deepEqual(totalPerDraw(counts, 3), [111, 222, 333])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalPerDraw: a single group is its own total', () => {
|
||||||
|
assert.deepEqual(totalPerDraw({ BL_M: [4, 5] }, 2), [4, 5])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalPerDraw: null when any group is short of nDraws', () => {
|
||||||
|
// Summing a 2-draw group into a 3-draw total would silently undercount the
|
||||||
|
// last draw, biasing the whole interval downward.
|
||||||
|
assert.equal(totalPerDraw({ BL_F: [1, 2, 3], BL_M: [1, 2] }, 3), null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalPerDraw: null when a group has a hole', () => {
|
||||||
|
const holey = [1, 2, 3]
|
||||||
|
delete holey[1]
|
||||||
|
assert.equal(totalPerDraw({ BL_F: holey }, 3), null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalPerDraw: null for empty or missing input', () => {
|
||||||
|
assert.equal(totalPerDraw({}, 500), null)
|
||||||
|
assert.equal(totalPerDraw(null, 500), null)
|
||||||
|
assert.equal(totalPerDraw({ BL_F: [1] }, 0), null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalPerDraw: does not mutate its input', () => {
|
||||||
|
const counts = { BL_F: [1, 2], BL_M: [3, 4] }
|
||||||
|
totalPerDraw(counts, 2)
|
||||||
|
assert.deepEqual(counts, { BL_F: [1, 2], BL_M: [3, 4] })
|
||||||
|
})
|
||||||
|
|
||||||
|
// ——— totalInterval ———
|
||||||
|
|
||||||
|
test('totalInterval: median and 95% bounds from the draw totals', () => {
|
||||||
|
// Symmetric and unimodal, so the HPD should sit close to the equal-tailed
|
||||||
|
// 25–975 without being required to equal it.
|
||||||
|
const totals = []
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
totals.push(500 + 120 * (Math.sin(i * 1.7) + Math.sin(i * 0.31) + Math.sin(i * 2.9)) / 3)
|
||||||
|
}
|
||||||
|
const iv = totalInterval(totals)
|
||||||
|
assert.equal(iv.nDraws, 1000)
|
||||||
|
assert.ok(iv.lower < iv.median && iv.median < iv.upper)
|
||||||
|
assert.ok(Math.abs(iv.median - 500) < 25, `median ${iv.median}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: bounds are the narrowest window covering 95% of draws', () => {
|
||||||
|
const totals = Array.from({ length: 1000 }, (_, i) => i)
|
||||||
|
const iv = totalInterval(totals)
|
||||||
|
const inside = totals.filter((t) => t >= iv.lower && t <= iv.upper).length
|
||||||
|
assert.ok(inside >= 950, `only ${inside} of 1000 draws inside`)
|
||||||
|
// A uniform has no denser region, so the narrowest window is ~95% of the range.
|
||||||
|
assert.ok(iv.upper - iv.lower <= 951, `width ${iv.upper - iv.lower}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hpdBounds: never wider than the equal-tailed interval', () => {
|
||||||
|
// Right-skewed, which is where the two definitions diverge most.
|
||||||
|
const skewed = Array.from({ length: 2000 }, (_, i) => Math.round(((i * 7919) % 1000) ** 1.6 / 1000))
|
||||||
|
const [lo, hi] = hpdBounds(skewed, 0.95)
|
||||||
|
const sorted = [...skewed].sort((a, b) => a - b)
|
||||||
|
const etLo = sorted[Math.floor(0.025 * (sorted.length - 1))]
|
||||||
|
const etHi = sorted[Math.ceil(0.975 * (sorted.length - 1))]
|
||||||
|
assert.ok(hi - lo <= etHi - etLo, `hpd ${hi - lo} vs equal-tailed ${etHi - etLo}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hpdBounds: stays in the dense region when it holds enough mass', () => {
|
||||||
|
// 970 draws at 0-9 and 30 stragglers out at 500. The cluster alone covers 97%,
|
||||||
|
// so the narrowest 95% window fits inside it and the far tail is excluded.
|
||||||
|
const values = [
|
||||||
|
...Array.from({ length: 970 }, (_, i) => i % 10),
|
||||||
|
...Array.from({ length: 30 }, () => 500),
|
||||||
|
]
|
||||||
|
const [lo, hi] = hpdBounds(values, 0.95)
|
||||||
|
assert.equal(lo, 0)
|
||||||
|
assert.ok(hi < 500, `upper bound ${hi} should exclude the far cluster`)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hpdBounds: still reaches the tail when the cluster is too small', () => {
|
||||||
|
// The mirror case, and the honest one: 90% in the cluster cannot cover a 95%
|
||||||
|
// interval, so the bound must extend outward rather than under-covering.
|
||||||
|
const values = [
|
||||||
|
...Array.from({ length: 900 }, (_, i) => i % 10),
|
||||||
|
...Array.from({ length: 100 }, () => 500),
|
||||||
|
]
|
||||||
|
const [, hi] = hpdBounds(values, 0.95)
|
||||||
|
assert.equal(hi, 500)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hpdBounds: degenerate input collapses to a point', () => {
|
||||||
|
assert.deepEqual(hpdBounds(new Array(100).fill(7), 0.95), [7, 7])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('hpdBounds: does not mutate its input', () => {
|
||||||
|
const values = [5, 1, 3]
|
||||||
|
hpdBounds(values, 0.95)
|
||||||
|
assert.deepEqual(values, [5, 1, 3])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: uses a 95% mass, matching the API convention', () => {
|
||||||
|
assert.equal(TOTAL_INTERVAL_MASS, 0.95)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: lower <= median <= upper', () => {
|
||||||
|
const totals = Array.from({ length: 500 }, (_, i) => (i * 7919) % 331)
|
||||||
|
const iv = totalInterval(totals)
|
||||||
|
assert.ok(iv.lower <= iv.median && iv.median <= iv.upper)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: a degenerate total collapses to a point', () => {
|
||||||
|
const iv = totalInterval(new Array(500).fill(12))
|
||||||
|
assert.deepEqual([iv.lower, iv.median, iv.upper], [12, 12, 12])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: null for empty or missing totals', () => {
|
||||||
|
assert.equal(totalInterval([]), null)
|
||||||
|
assert.equal(totalInterval(null), null)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('totalInterval: is narrower than summing the groups own bounds', () => {
|
||||||
|
// The property that motivates this whole path. Two independent groups, each
|
||||||
|
// roughly uniform on 0..100: summing each group's 97.5th percentile gives
|
||||||
|
// ~200, but the 97.5th percentile of the *sum* is well below that, because
|
||||||
|
// both groups landing at their extreme in the same draw is rare.
|
||||||
|
const n = 4000
|
||||||
|
const a = Array.from({ length: n }, (_, i) => (i * 37) % 101)
|
||||||
|
const b = Array.from({ length: n }, (_, i) => (i * 61) % 101)
|
||||||
|
const iv = totalInterval(totalPerDraw({ A: a, B: b }, n))
|
||||||
|
const summedBounds = { lower: 0, upper: 100 + 100 }
|
||||||
|
assert.ok(iv.upper < summedBounds.upper, `sum-of-quantiles ${summedBounds.upper} vs quantile-of-sum ${iv.upper}`)
|
||||||
|
assert.ok(iv.upper - iv.lower < summedBounds.upper - summedBounds.lower)
|
||||||
|
})
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
/**
|
||||||
|
* Collision-free placement for direct labels on a shared axis.
|
||||||
|
*
|
||||||
|
* The density panel labels each curve at its own peak rather than shipping a
|
||||||
|
* legend, which only works if the labels don't collide — and in this app they
|
||||||
|
* collide constantly, because the interesting districts are exactly the ones
|
||||||
|
* where several groups have similar rates. Worse, each density is normalized to
|
||||||
|
* its own peak height, so every curve peaks at the *same* y and a naive
|
||||||
|
* placement puts every label on one line: three overlapping groups rendered as
|
||||||
|
* "HiWhite:F: Black F".
|
||||||
|
*
|
||||||
|
* So labels are packed into horizontal lanes: first-fit by x, dropping to a new
|
||||||
|
* lane only when the previous one is occupied at that position. Groups that are
|
||||||
|
* far apart still share a lane, which keeps the common case compact.
|
||||||
|
*
|
||||||
|
* Pure geometry — no DOM, no measurement. SVG text can't be measured before
|
||||||
|
* render, so widths are estimated from character count; the estimate is
|
||||||
|
* deliberately generous so labels err toward extra separation rather than
|
||||||
|
* overlap.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Mean advance width of a glyph as a fraction of font size, for the app's sans
|
||||||
|
// stack at the weights these labels use. Slightly over the true average so the
|
||||||
|
// packing errs toward separation.
|
||||||
|
const CHAR_WIDTH_RATIO = 0.62
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} text
|
||||||
|
* @param {number} fontPx
|
||||||
|
* @returns {number} approximate rendered width in px
|
||||||
|
*/
|
||||||
|
export function estimateTextWidth(text, fontPx) {
|
||||||
|
return (text || '').length * fontPx * CHAR_WIDTH_RATIO
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {Array<{key: string, x: number, text: string}>} items - one per label,
|
||||||
|
* `x` being the point it wants to sit above (a curve's peak).
|
||||||
|
* @param {{min: number, max: number, fontPx: number, gap?: number}} options -
|
||||||
|
* `min`/`max` are the plot's horizontal bounds; labels are kept inside them.
|
||||||
|
* @returns {{labels: Array<{key: string, text: string, x: number,
|
||||||
|
* anchor: 'start'|'middle'|'end', width: number, left: number, right: number,
|
||||||
|
* lane: number}>, lanes: number}}
|
||||||
|
* `labels` is in input order; `lanes` is how many rows the caller must
|
||||||
|
* reserve above the plot.
|
||||||
|
*/
|
||||||
|
export function layoutPeakLabels(items, { min, max, fontPx, gap = 6 } = {}) {
|
||||||
|
if (!items?.length) return { labels: [], lanes: 0 }
|
||||||
|
|
||||||
|
const placed = items.map((item) => {
|
||||||
|
const width = estimateTextWidth(item.text, fontPx)
|
||||||
|
const half = width / 2
|
||||||
|
|
||||||
|
// Anchor outward near the edges so a centred label can't overflow the plot.
|
||||||
|
let anchor = 'middle'
|
||||||
|
if (item.x - half < min) anchor = 'start'
|
||||||
|
else if (item.x + half > max) anchor = 'end'
|
||||||
|
|
||||||
|
// Clamp the anchor point itself, so a peak clipped to the axis edge still
|
||||||
|
// yields a label fully inside the frame.
|
||||||
|
let x = item.x
|
||||||
|
if (anchor === 'start') x = Math.max(min, Math.min(x, max - width))
|
||||||
|
else if (anchor === 'end') x = Math.min(max, Math.max(x, min + width))
|
||||||
|
else x = Math.min(Math.max(x, min + half), max - half)
|
||||||
|
|
||||||
|
const left = anchor === 'start' ? x : anchor === 'end' ? x - width : x - half
|
||||||
|
return { ...item, width, anchor, x, left, right: left + width, lane: 0 }
|
||||||
|
})
|
||||||
|
|
||||||
|
// First-fit by left edge. Sorting only decides lane order; the returned array
|
||||||
|
// keeps the caller's original order.
|
||||||
|
const laneRightEdge = []
|
||||||
|
for (const label of [...placed].sort((a, b) => a.left - b.left)) {
|
||||||
|
let lane = 0
|
||||||
|
while (lane < laneRightEdge.length && laneRightEdge[lane] + gap > label.left) lane++
|
||||||
|
laneRightEdge[lane] = label.right
|
||||||
|
label.lane = lane
|
||||||
|
}
|
||||||
|
|
||||||
|
return { labels: placed, lanes: laneRightEdge.length }
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
import { test } from 'node:test'
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import { estimateTextWidth, layoutPeakLabels } from './labelLayout.js'
|
||||||
|
|
||||||
|
const BOUNDS = { min: 0, max: 400, fontPx: 10 }
|
||||||
|
|
||||||
|
test('estimateTextWidth: grows with text length and font size', () => {
|
||||||
|
assert.ok(estimateTextWidth('AB', 10) > estimateTextWidth('A', 10))
|
||||||
|
assert.ok(estimateTextWidth('ABC', 20) > estimateTextWidth('ABC', 10))
|
||||||
|
assert.ok(estimateTextWidth('', 10) >= 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: well-separated labels all sit on lane 0', () => {
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[{ key: 'a', x: 20, text: 'A' }, { key: 'b', x: 200, text: 'B' }, { key: 'c', x: 380, text: 'C' }],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
assert.deepEqual(out.labels.map((l) => l.lane), [0, 0, 0])
|
||||||
|
assert.equal(out.lanes, 1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: overlapping labels are pushed to separate lanes', () => {
|
||||||
|
// This is the failing case from the live site: three densities peaking at
|
||||||
|
// nearly the same rate rendered as "HiWhite:F: Black F".
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[
|
||||||
|
{ key: 'hi', x: 100, text: 'Hispanic F' },
|
||||||
|
{ key: 'wh', x: 104, text: 'White F' },
|
||||||
|
{ key: 'bl', x: 108, text: 'Black F' },
|
||||||
|
],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
const lanes = out.labels.map((l) => l.lane).sort()
|
||||||
|
assert.deepEqual(lanes, [0, 1, 2])
|
||||||
|
assert.equal(out.lanes, 3)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: identical positions never share a lane', () => {
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[
|
||||||
|
{ key: 'a', x: 200, text: 'Hispanic F' },
|
||||||
|
{ key: 'b', x: 200, text: 'Hispanic M' },
|
||||||
|
],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
assert.notEqual(out.labels[0].lane, out.labels[1].lane)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: a lane is reused once there is horizontal room', () => {
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[
|
||||||
|
{ key: 'a', x: 20, text: 'A' },
|
||||||
|
{ key: 'b', x: 24, text: 'B' },
|
||||||
|
{ key: 'c', x: 380, text: 'C' },
|
||||||
|
],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
const byKey = Object.fromEntries(out.labels.map((l) => [l.key, l.lane]))
|
||||||
|
assert.equal(byKey.a, 0)
|
||||||
|
assert.equal(byKey.b, 1)
|
||||||
|
// 'c' is far away, so it drops back to the first lane rather than stacking.
|
||||||
|
assert.equal(byKey.c, 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: anchors outward at the edges so text stays in frame', () => {
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[{ key: 'l', x: 0, text: 'Hispanic F' }, { key: 'r', x: 400, text: 'Hispanic F' }],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
const byKey = Object.fromEntries(out.labels.map((l) => [l.key, l]))
|
||||||
|
assert.equal(byKey.l.anchor, 'start')
|
||||||
|
assert.equal(byKey.r.anchor, 'end')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: a mid-plot label stays centred on its peak', () => {
|
||||||
|
const out = layoutPeakLabels([{ key: 'm', x: 200, text: 'Black F' }], BOUNDS)
|
||||||
|
assert.equal(out.labels[0].anchor, 'middle')
|
||||||
|
assert.equal(out.labels[0].x, 200)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: no label extends outside the plot bounds', () => {
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[
|
||||||
|
{ key: 'l', x: -50, text: 'American Indian / Alaska Native' },
|
||||||
|
{ key: 'r', x: 900, text: 'American Indian / Alaska Native' },
|
||||||
|
],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
for (const l of out.labels) {
|
||||||
|
assert.ok(l.left >= BOUNDS.min - 0.01, `${l.key} left ${l.left} < ${BOUNDS.min}`)
|
||||||
|
assert.ok(l.right <= BOUNDS.max + 0.01, `${l.key} right ${l.right} > ${BOUNDS.max}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: preserves input order in the output', () => {
|
||||||
|
// Lane assignment sorts internally; callers still key off their own order.
|
||||||
|
const out = layoutPeakLabels(
|
||||||
|
[{ key: 'z', x: 300, text: 'Z' }, { key: 'a', x: 10, text: 'A' }],
|
||||||
|
BOUNDS,
|
||||||
|
)
|
||||||
|
assert.deepEqual(out.labels.map((l) => l.key), ['z', 'a'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: empty input yields zero lanes', () => {
|
||||||
|
const out = layoutPeakLabels([], BOUNDS)
|
||||||
|
assert.deepEqual(out.labels, [])
|
||||||
|
assert.equal(out.lanes, 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('layoutPeakLabels: wider gap forces more lanes', () => {
|
||||||
|
const items = [{ key: 'a', x: 100, text: 'A' }, { key: 'b', x: 130, text: 'B' }]
|
||||||
|
const tight = layoutPeakLabels(items, { ...BOUNDS, gap: 0 })
|
||||||
|
const loose = layoutPeakLabels(items, { ...BOUNDS, gap: 40 })
|
||||||
|
assert.ok(loose.lanes > tight.lanes)
|
||||||
|
})
|
||||||
+17
-2
@@ -10,15 +10,22 @@ import { quantile } from './kde.js'
|
|||||||
import { niceTicks } from './niceTicks.js'
|
import { niceTicks } from './niceTicks.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Hard ceiling on the axis, inherited from the chart this replaces.
|
* Hard ceiling on the axis.
|
||||||
*
|
*
|
||||||
* A four-student cell can produce posterior draws in the hundreds per 1,000;
|
* A four-student cell can produce posterior draws in the hundreds per 1,000;
|
||||||
* letting those set the axis squashes every other group into the leftmost few
|
* letting those set the axis squashes every other group into the leftmost few
|
||||||
* pixels. `computeRateDomain` reports `clipped` whenever the cap actually
|
* pixels. `computeRateDomain` reports `clipped` whenever the cap actually
|
||||||
* binds, so the chart can say so rather than cropping data off the edge in
|
* binds, so the chart can say so rather than cropping data off the edge in
|
||||||
* silence.
|
* silence.
|
||||||
|
*
|
||||||
|
* Set to 100 per 1,000 (one arrest per ten students). The previous value of
|
||||||
|
* 30 was exceeded by over 60% of districts — mostly sparse groups with small
|
||||||
|
* enrollment cells whose Agresti–Coull upper bounds genuinely extend that far.
|
||||||
|
* A cap of 100 still guards against the most degenerate cases (e.g. a single
|
||||||
|
* predicted arrest in a four-student cell yielding ~250 per 1,000) while
|
||||||
|
* letting realistic data drive the axis for the vast majority of districts.
|
||||||
*/
|
*/
|
||||||
export const MAX_RATE_DOMAIN = 30
|
export const MAX_RATE_DOMAIN = 100
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ignore the very top of each draw set when sizing the axis. The posterior
|
* Ignore the very top of each draw set when sizing the axis. The posterior
|
||||||
@@ -30,6 +37,14 @@ const DOMAIN_QUANTILE = 0.995
|
|||||||
const HEADROOM = 1.15
|
const HEADROOM = 1.15
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* Computes a shared x-domain for the rate density and over-time charts.
|
||||||
|
*
|
||||||
|
* The domain covers everything that will actually be drawn: the 0.995
|
||||||
|
* quantile of each group's posterior-predictive draw rates, plus every
|
||||||
|
* Agresti–Coull upper bound from selected groups. `niceMax` is a clean
|
||||||
|
* "nice number" ceiling (via `niceTicks`) that may exceed the raw data max by
|
||||||
|
* up to one tick step, and is capped at `MAX_RATE_DOMAIN`.
|
||||||
|
*
|
||||||
* @param {Array<number[]>} rateArrays - per-group draw rates, per 1,000
|
* @param {Array<number[]>} rateArrays - per-group draw rates, per 1,000
|
||||||
* @param {number[]} acUpperRates - Agresti–Coull upper bounds, per 1,000
|
* @param {number[]} acUpperRates - Agresti–Coull upper bounds, per 1,000
|
||||||
* @returns {{ticks: number[], niceMax: number, clipped: boolean}}
|
* @returns {{ticks: number[], niceMax: number, clipped: boolean}}
|
||||||
|
|||||||
@@ -23,16 +23,27 @@ test('computeRateDomain: one extreme draw does not blow out the axis', () => {
|
|||||||
assert.ok(domain.niceMax < 10, `niceMax was ${domain.niceMax}`)
|
assert.ok(domain.niceMax < 10, `niceMax was ${domain.niceMax}`)
|
||||||
})
|
})
|
||||||
|
|
||||||
test('computeRateDomain: caps the domain and reports the clip', () => {
|
test('computeRateDomain: caps at MAX_RATE_DOMAIN=100 and reports the clip', () => {
|
||||||
// The cap keeps a tiny-denominator group from flattening every other curve,
|
// The cap keeps a tiny-denominator group from flattening every other curve,
|
||||||
// but the caller has to be able to say so on screen rather than silently
|
// but the caller has to be able to say so on screen rather than silently
|
||||||
// cropping data off the right edge.
|
// cropping data off the right edge.
|
||||||
const wide = new Array(500).fill(0).map((_, i) => (i < 490 ? 60 : 0))
|
const wide = new Array(500).fill(0).map((_, i) => (i < 490 ? 200 : 0))
|
||||||
const domain = computeRateDomain([wide], [80])
|
const domain = computeRateDomain([wide], [300])
|
||||||
assert.equal(domain.niceMax, MAX_RATE_DOMAIN)
|
assert.equal(domain.niceMax, MAX_RATE_DOMAIN)
|
||||||
assert.equal(domain.clipped, true)
|
assert.equal(domain.clipped, true)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('computeRateDomain: realistic sparse district data is not clipped', () => {
|
||||||
|
// Carson City NV (LEAID 3200390): pooled BL cell with 34 students and 0
|
||||||
|
// observed arrests → AC upper bound ≈ 88 per 1,000. This should NOT be
|
||||||
|
// clipped under the new cap of 100.
|
||||||
|
const blRates = new Array(500).fill(0)
|
||||||
|
const domain = computeRateDomain([blRates], [88])
|
||||||
|
assert.ok(domain.niceMax >= 88, `niceMax ${domain.niceMax} should cover AC bound of 88`)
|
||||||
|
assert.equal(domain.clipped, false)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
test('computeRateDomain: never returns a zero-width domain', () => {
|
test('computeRateDomain: never returns a zero-width domain', () => {
|
||||||
const domain = computeRateDomain([[0, 0, 0]], [0])
|
const domain = computeRateDomain([[0, 0, 0]], [0])
|
||||||
assert.ok(domain.niceMax > 0)
|
assert.ok(domain.niceMax > 0)
|
||||||
@@ -49,3 +60,14 @@ test('computeRateDomain: ignores empty rate arrays', () => {
|
|||||||
const domain = computeRateDomain([[], [1, 2, 3]], [])
|
const domain = computeRateDomain([[], [1, 2, 3]], [])
|
||||||
assert.ok(domain.niceMax > 0 && domain.niceMax < MAX_RATE_DOMAIN)
|
assert.ok(domain.niceMax > 0 && domain.niceMax < MAX_RATE_DOMAIN)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('computeRateDomain: degenerate tiny-denominator cell is clipped', () => {
|
||||||
|
// A four-student cell with a single predicted arrest yields ~250 per
|
||||||
|
// 1,000 — this should be capped at MAX_RATE_DOMAIN=100 and flagged as
|
||||||
|
// clipped so the chart can warn the reader.
|
||||||
|
const tinyRates = new Array(500).fill(0)
|
||||||
|
for (let i = 0; i < 250; i++) tinyRates[i] = 250
|
||||||
|
const domain = computeRateDomain([tinyRates], [300])
|
||||||
|
assert.equal(domain.niceMax, MAX_RATE_DOMAIN)
|
||||||
|
assert.equal(domain.clipped, true)
|
||||||
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user