fix: raise MAX_RATE_DOMAIN from 30 to 100 per 1,000 students
Deploy to git-pages / deploy (push) Successful in 19s
Deploy to git-pages / deploy (push) Successful in 19s
The previous cap of 30 was exceeded by over 60% of districts — mostly sparse groups with small enrollment cells whose Agresti-Coull upper bounds genuinely extend that far. Analysis across all 51 states (16,279 districts) showed the median max x-value is already ~56 per 1,000; only truly degenerate cases like a single predicted arrest in a four-student cell (~250/1000) need capping. A cap of 100 still guards against these outliers while letting realistic data drive the axis for the vast majority of districts. The existing 'clipped' flag and note mechanism remain unchanged — they activate only when extreme values are encountered.
This commit is contained in:
@@ -170,8 +170,8 @@ time, the port does not).
|
||||
posterior predictive is drawn as discrete mass, because it *is* discrete; a
|
||||
Gaussian KDE over four achievable values renders as a lumpy smear that reads
|
||||
as a rendering bug.
|
||||
- **`src/utils/rateDomain.js`** — `MAX_RATE_DOMAIN = 30` caps the axis so a
|
||||
four-student cell can't squash every other curve. It reports `clipped` so the
|
||||
- **`src/utils/rateDomain.js`** — `MAX_RATE_DOMAIN = 100` caps the axis so a
|
||||
four-student cell can't squash every other curve (up from 30, which was exceeded by over 60% of districts). It reports `clipped` so the
|
||||
chart says so instead of silently cropping.
|
||||
|
||||
## Common Pitfalls & Gotchas
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
/**
|
||||
* Analysis: distribution of maximum x-axis values for the rate density chart.
|
||||
*
|
||||
* The "Arrest rate probability density" chart currently caps its x-axis at a
|
||||
* hard-coded MAX_RATE_DOMAIN = 30 (per 1,000 students). We want to understand
|
||||
* what the true distribution of max x-values is across districts so we can
|
||||
* decide whether to raise or make this cap dynamic.
|
||||
*
|
||||
* What feeds computeRateDomain():
|
||||
* 1. Per-group rate arrays from real posterior draws → 0.995 quantile of each
|
||||
* (requires parquet data; approximated here via model estimates)
|
||||
* 2. Agresti-Coull upper bound rates = (ac.upper / enroll) * 1000, computed
|
||||
* from observed counts and enrollment — available via the API for every
|
||||
* race×sex cell in every district
|
||||
*
|
||||
* KEY INSIGHT: Only SELECTED groups feed into computeRateDomain. Per
|
||||
* `defaultSelectedKeys` in districtGroups.js, only groups with ≥1 observed
|
||||
* arrest are selected by default (or top-2 by enrollment if none have arrests).
|
||||
* Additionally, when total district arrests < 20, sex pooling merges F+M cells,
|
||||
* which combines enrollment and observed counts per race.
|
||||
*/
|
||||
|
||||
const BASE = 'https://crdc-api.civilytics.org/api/v1'
|
||||
const LIMIT = 500
|
||||
const POOL_THRESHOLD = 20 // POOL_BY_SEX_ARREST_THRESHOLD from pooling.js
|
||||
|
||||
/** Fetch JSON from the API, retrying on transient failures. */
|
||||
async function apiFetch(path) {
|
||||
const url = `${BASE}${path}`
|
||||
let lastErr
|
||||
for (let attempt = 0; attempt <= 3; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(30000) })
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||
return await res.json()
|
||||
} catch (err) {
|
||||
lastErr = err
|
||||
if (attempt < 3) {
|
||||
const wait = 500 * Math.pow(2, attempt) + Math.random() * 100
|
||||
console.error(` retry ${attempt + 1}/3 after ${Math.round(wait)}ms:`, err.message)
|
||||
await new Promise((r) => setTimeout(r, wait))
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr
|
||||
}
|
||||
|
||||
/** Peter Acklam's inverse normal CDF (probit), ~1.15e-9 relative error. */
|
||||
function probit(p) {
|
||||
const a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02,
|
||||
1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]
|
||||
const b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02,
|
||||
6.680131188771972e+01, -1.328068155288572e+01]
|
||||
const c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00,
|
||||
-2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]
|
||||
const d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00,
|
||||
3.754408661907416e+00]
|
||||
const pLow = 0.02425, pHigh = 1 - pLow
|
||||
|
||||
if (p < pLow) {
|
||||
const q = Math.sqrt(-2 * Math.log(p))
|
||||
return (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /
|
||||
((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
|
||||
}
|
||||
if (p <= pHigh) {
|
||||
const q = p - 0.5, r = q * q
|
||||
return (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5]) * q /
|
||||
(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1)
|
||||
}
|
||||
const q = Math.sqrt(-2 * Math.log(1 - p))
|
||||
return -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /
|
||||
((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
|
||||
}
|
||||
|
||||
/** Agresti-Coull upper bound (count scale), port of agrestiCoull.js. */
|
||||
function acUpperBound(numerator, denominator, confidenceLevel = 0.95) {
|
||||
const adjStar = probit(1 - (1 - confidenceLevel) / 2)
|
||||
if (numerator > 0) {
|
||||
const numStar = numerator + adjStar
|
||||
const denomStar = denominator + 2 * adjStar
|
||||
const phat = numStar / denomStar
|
||||
const se = Math.sqrt((phat / denomStar) * (1 - phat))
|
||||
return (phat + adjStar * se) * denomStar
|
||||
}
|
||||
// Zero events: rule of three — upper bound ≈ 3 regardless of enrollment.
|
||||
return -Math.log(1 - confidenceLevel)
|
||||
}
|
||||
|
||||
/** Estimate the 0.995 quantile of posterior predictive draw rates. */
|
||||
function estimateDrawQuantile(row, targetP = 0.995) {
|
||||
const enroll = row.stu_enroll || 0
|
||||
if (enroll <= 0) return null
|
||||
|
||||
// When count_upper is available and > 0, the model's posterior predictive
|
||||
// upper bound gives a sense of the spread. For sparse groups with few or zero
|
||||
// arrests, draw quantiles can be extreme because observation noise dominates:
|
||||
// a single predicted arrest in a small cell produces a huge per-1000 rate.
|
||||
const countUpper = row.count_upper || 0
|
||||
if (countUpper > 0) {
|
||||
return (countUpper / enroll) * 1000
|
||||
}
|
||||
|
||||
// When count_upper is 0, use the Agresti-Coull upper bound as a conservative
|
||||
// proxy — it's an honest frequentist interval that also tends to be extreme
|
||||
// for sparse groups.
|
||||
const ac = acUpperBound(row.observed_arrests || 0, enroll)
|
||||
return (ac / enroll) * 1000
|
||||
}
|
||||
|
||||
/** Determine which race×sex cells are "selected" by default per districtGroups.js logic. */
|
||||
function determineSelectedCells(cells, pooled) {
|
||||
const usable = cells.filter(
|
||||
(r) => ['WH', 'BL', 'HI', 'AM'].includes(r.race) && ['F', 'M'].includes(r.sex),
|
||||
)
|
||||
|
||||
if (pooled) {
|
||||
// When pooled: merge F+M per race, then select groups with ≥1 arrest or top-2 by enrollment
|
||||
const races = {}
|
||||
for (const r of usable) {
|
||||
if (!races[r.race]) races[r.race] = { race: r.race, enroll: 0, observed: 0 }
|
||||
races[r.race].enroll += r.stu_enroll || 0
|
||||
races[r.race].observed += r.observed_arrests || 0
|
||||
}
|
||||
const merged = Object.values(races)
|
||||
|
||||
// defaultSelectedKeys logic on pooled groups
|
||||
const withArrests = merged.filter((r) => r.observed > 0)
|
||||
if (withArrests.length > 0) {
|
||||
return withArrests.map((r) => ({ race: r.race, sex: null, enroll: r.enroll, observed: r.observed }))
|
||||
}
|
||||
return [...merged]
|
||||
.sort((a, b) => b.enroll - a.enroll)
|
||||
.slice(0, 2)
|
||||
.map((r) => ({ race: r.race, sex: null, enroll: r.enroll, observed: r.observed }))
|
||||
} else {
|
||||
// Unpooled: select groups with ≥1 arrest or top-2 by enrollment
|
||||
const withArrests = usable.filter((r) => (r.observed_arrests || 0) > 0)
|
||||
if (withArrests.length > 0) return withArrests
|
||||
|
||||
return [...usable]
|
||||
.sort((a, b) => (b.stu_enroll || 0) - (a.stu_enroll || 0))
|
||||
.slice(0, 2)
|
||||
}
|
||||
}
|
||||
|
||||
/** Compute max x-value candidate from selected cells. */
|
||||
function computeMaxX(selectedCells, allCells, pooled) {
|
||||
let maxAcRate = 0
|
||||
let maxDrawEstimate = 0
|
||||
|
||||
for (const cell of selectedCells) {
|
||||
const enroll = cell.enroll || 0
|
||||
if (enroll <= 0) continue
|
||||
const observed = cell.observed || 0
|
||||
|
||||
// AC upper bound rate (always a candidate in computeRateDomain)
|
||||
const acUpper = acUpperBound(observed, enroll)
|
||||
const acRate = (acUpper / enroll) * 1000
|
||||
if (acRate > maxAcRate) maxAcRate = acRate
|
||||
|
||||
// Estimated draw quantile — find the matching row in allCells for model estimates
|
||||
let countUpper = 0
|
||||
let foundRow = null
|
||||
if (!pooled && cell.sex) {
|
||||
foundRow = allCells.find((r) => r.race === cell.race && r.sex === cell.sex)
|
||||
} else if (pooled) {
|
||||
// For pooled, find the row with max count_upper for this race across both sexes
|
||||
const raceRows = allCells.filter((r) => r.race === cell.race && ['F', 'M'].includes(r.sex))
|
||||
for (const rr of raceRows) {
|
||||
if ((rr.count_upper || 0) > countUpper) countUpper = rr.count_upper || 0
|
||||
}
|
||||
}
|
||||
|
||||
if (!pooled && foundRow) {
|
||||
countUpper = foundRow.count_upper || 0
|
||||
}
|
||||
|
||||
const enroll_ = cell.enroll || 1
|
||||
let drawEst = null
|
||||
if (countUpper > 0) {
|
||||
drawEst = (countUpper / enroll_) * 1000
|
||||
} else {
|
||||
// Conservative proxy via AC upper bound
|
||||
const ac = acUpperBound(observed, enroll_)
|
||||
drawEst = (ac / enroll_) * 1000
|
||||
}
|
||||
|
||||
if (drawEst > maxDrawEstimate) maxDrawEstimate = drawEst
|
||||
}
|
||||
|
||||
return Math.max(maxAcRate, maxDrawEstimate)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('=== Rate Domain Analysis ===\n')
|
||||
|
||||
// Step 1: Get all states from the API
|
||||
const statesResp = await apiFetch('/states?limit=500')
|
||||
const stateSet = new Set(statesResp.data.map((r) => r.state))
|
||||
const states = [...stateSet].sort()
|
||||
console.log(`Found ${states.length} states\n`)
|
||||
|
||||
// Step 2: For each state, page through all estimates and collect per-district data
|
||||
const districtData = new Map() // leaid -> { state, name, cells: [] }
|
||||
let totalRows = 0
|
||||
|
||||
for (const state of states) {
|
||||
console.log(`Fetching ${state}...`)
|
||||
const firstResp = await apiFetch(`/estimates?state=${state}&limit=${LIMIT}`)
|
||||
const total = firstResp.meta.total
|
||||
const nPages = Math.ceil(total / LIMIT)
|
||||
|
||||
let rows = [...firstResp.data]
|
||||
for (let page = 1; page < nPages; page++) {
|
||||
process.stderr.write(` ${state} page ${page + 1}/${nPages}\r`)
|
||||
const resp = await apiFetch(`/estimates?state=${state}&limit=${LIMIT}&page=${page}`)
|
||||
rows.push(...resp.data)
|
||||
}
|
||||
totalRows += rows.length
|
||||
|
||||
for (const row of rows) {
|
||||
const key = `${row.state}|${row.leaid}`
|
||||
if (!districtData.has(key)) {
|
||||
districtData.set(key, { state: row.state, leaid: row.leaid, name: row.lea_name, cells: [] })
|
||||
}
|
||||
districtData.get(key).cells.push(row)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nTotal rows fetched: ${totalRows}`)
|
||||
console.log(`Total districts: ${districtData.size}\n`)
|
||||
|
||||
// Step 3: For each district, compute the max x-value for both pooled and unpooled modes
|
||||
const results = []
|
||||
let nPooled = 0
|
||||
let nUnpooled = 0
|
||||
|
||||
for (const [key, dist] of districtData) {
|
||||
const totalArrests = dist.cells.reduce((sum, r) => sum + (r.observed_arrests || 0), 0)
|
||||
const pooled = totalArrests < POOL_THRESHOLD
|
||||
if (pooled) nPooled++
|
||||
else nUnpooled++
|
||||
|
||||
let maxX
|
||||
if (pooled) {
|
||||
// Pooled mode: AC bounds computed per race (F+M merged). Note that for
|
||||
// pooled groups, the app shows a note but still computes AC bounds.
|
||||
const selected = determineSelectedCells(dist.cells, true)
|
||||
maxX = computeMaxX(selected, dist.cells, true)
|
||||
} else {
|
||||
const selected = determineSelectedCells(dist.cells, false)
|
||||
maxX = computeMaxX(selected, dist.cells, false)
|
||||
}
|
||||
|
||||
let totalEnroll = 0
|
||||
for (const cell of dist.cells) {
|
||||
if ((cell.stu_enroll || 0) > 0) totalEnroll += cell.stu_enroll
|
||||
}
|
||||
|
||||
results.push({
|
||||
key, state: dist.state, leaid: dist.leaid, name: dist.name,
|
||||
totalEnroll, pooled, maxX: maxX * 1.15, // HEADROOM factor from rateDomain.js
|
||||
})
|
||||
}
|
||||
|
||||
console.log(`Districts with sex pooling (total arrests < ${POOL_THRESHOLD}): ${nPooled} (${(nPooled / results.length * 100).toFixed(1)}%)`)
|
||||
console.log(`Districts without pooling: ${nUnpooled} (${(nUnpooled / results.length * 100).toFixed(1)}%)\n`)
|
||||
|
||||
// Step 4: Analyze distribution of max x-values
|
||||
const sorted = results.sort((a, b) => a.maxX - b.maxX)
|
||||
const n = sorted.length
|
||||
|
||||
console.log('=== Distribution of Maximum X-Values (per 1,000 students) ===\n')
|
||||
|
||||
// Summary statistics — maxX already includes HEADROOM(1.15) factor
|
||||
const percentiles = [5, 10, 25, 50, 75, 90, 95, 99, 99.9]
|
||||
console.log('Percentiles of max x-value (includes HEADROOM=1.15):')
|
||||
for (const p of percentiles) {
|
||||
const idx = Math.floor((p / 100) * (n - 1))
|
||||
console.log(` ${p.toFixed(1)}th: ${sorted[idx].maxX.toFixed(2)} per 1,000`)
|
||||
}
|
||||
|
||||
console.log('\n--- Threshold analysis ---')
|
||||
const thresholds = [30, 40, 50, 60, 70, 80, 90, 100]
|
||||
for (const thresh of thresholds) {
|
||||
const count = sorted.filter((r) => r.maxX > thresh).length
|
||||
const pct = (count / n) * 100
|
||||
console.log(` Exceeds ${thresh}: ${count} districts (${pct.toFixed(2)}%)`)
|
||||
}
|
||||
|
||||
// Step 5: Break down by pooling status
|
||||
console.log('\n--- By pooling status ---')
|
||||
const pooledResults = sorted.filter((r) => r.pooled)
|
||||
const unpooledResults = sorted.filter((r) => !r.pooled)
|
||||
for (const thresh of [30, 50, 100]) {
|
||||
const pClipped = pooledResults.filter((r) => r.maxX > thresh).length
|
||||
const uClipped = unpooledResults.filter((r) => r.maxX > thresh).length
|
||||
console.log(` Cap=${thresh}: pooled ${pClipped}/${pooledResults.length} (${(pClipped / pooledResults.length * 100).toFixed(1)}%), ` +
|
||||
`unpooled ${uClipped}/${unpooledResults.length} (${(uClipped / unpooledResults.length * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
// Step 6: Show top districts by max x-value, with enrollment context
|
||||
console.log('\n--- Top 25 districts by max x-value ---')
|
||||
const top = sorted.slice(-25).reverse()
|
||||
for (const r of top) {
|
||||
const pooledStr = r.pooled ? ' [pooled]' : ''
|
||||
const clippedAt30 = r.maxX > 30 ? ' *** CLIPPED at 30' : ''
|
||||
console.log(` ${r.state} | LEAID ${r.leaid} | enroll=${r.totalEnroll.toLocaleString()}${pooledStr} | ` +
|
||||
`max_x=${r.maxX.toFixed(2)}/1000` + clippedAt30)
|
||||
}
|
||||
|
||||
// Step 7: Show realistic-size districts (enrollment > 500) that are clipped
|
||||
console.log('\n--- Clipped districts with enrollment > 500 ---')
|
||||
const realClipped = sorted.filter((r) => r.maxX > 30 && r.totalEnroll >= 500).sort((a, b) => b.maxX - a.maxX).slice(0, 15)
|
||||
for (const r of realClipped) {
|
||||
const pooledStr = r.pooled ? ' [pooled]' : ''
|
||||
console.log(` ${r.state} | ${r.name} | enroll=${r.totalEnroll.toLocaleString()}${pooledStr} | ` +
|
||||
`max_x=${r.maxX.toFixed(2)}/1000`)
|
||||
}
|
||||
|
||||
// Step 8: Non-clipped districts for context
|
||||
const notClipped = sorted.filter((r) => r.maxX <= 30).sort((a, b) => a.maxX - b.maxX)
|
||||
console.log(`\n--- Non-clipped districts (max_x ≤ 30): ${notClipped.length} (${(notClipped.length / n * 100).toFixed(2)}%) ---`)
|
||||
if (notClipped.length > 0) {
|
||||
const midIdx = Math.floor(notClipped.length / 2)
|
||||
console.log(' Sample non-clipped districts:')
|
||||
for (let i = Math.max(0, midIdx - 3); i < Math.min(notClipped.length, midIdx + 4); i++) {
|
||||
const r = notClipped[i]
|
||||
console.log(` ${r.state} | ${r.name} | enroll=${r.totalEnroll.toLocaleString()} | max_x=${r.maxX.toFixed(2)}/1000`)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 9: State-level summary (focusing on clipped counts)
|
||||
console.log('\n--- Top 15 states by % of districts clipped ---')
|
||||
const byState = {}
|
||||
for (const r of results) {
|
||||
if (!byState[r.state]) byState[r.state] = []
|
||||
byState[r.state].push(r.maxX)
|
||||
}
|
||||
const stateStats = Object.entries(byState).map(([state, vals]) => ({
|
||||
state, n: vals.length, median: percentile(vals, 50), p95: percentile(vals, 95),
|
||||
max: Math.max(...vals), clipped: vals.filter((v) => v > 30).length,
|
||||
})).sort((a, b) => (b.clipped / b.n) - (a.clipped / a.n))
|
||||
|
||||
for (const s of stateStats.slice(0, 15)) {
|
||||
console.log(` ${s.state}: n=${s.n}, median=${s.median.toFixed(1)}, p95=${s.p95.toFixed(1)}, ` +
|
||||
`max=${s.max.toFixed(1)}, clipped>30: ${s.clipped} (${(s.clipped / s.n * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
// Step 10: Recommendation analysis — what cap would minimize clipping while staying bounded?
|
||||
console.log('\n=== RECOMMENDATION ANALYSIS ===')
|
||||
const capOptions = [30, 40, 50, 60, 75, 100]
|
||||
for (const cap of capOptions) {
|
||||
const clipped = sorted.filter((r) => r.maxX > cap).length
|
||||
console.log(` Cap=${cap}: ${clipped} districts clipped (${(clipped / n * 100).toFixed(2)}%)`)
|
||||
}
|
||||
|
||||
// Step 11: Key findings summary
|
||||
console.log('\n--- Key Findings ---')
|
||||
const pctExceed30 = (sorted.filter((r) => r.maxX > 30).length / n) * 100
|
||||
const pctExceed50 = (sorted.filter((r) => r.maxX > 50).length / n) * 100
|
||||
console.log(`- ${pctExceed30.toFixed(2)}% of districts have a max x-value exceeding the current cap of 30`)
|
||||
console.log(`- ${pctExceed50.toFixed(2)}% exceed 50 per 1,000`)
|
||||
const p99 = sorted[Math.floor(0.99 * (n - 1))].maxX
|
||||
const max = sorted[n - 1].maxX
|
||||
console.log(`- 99th percentile: ${p99.toFixed(2)} per 1,000`)
|
||||
console.log(`- Maximum observed: ${max.toFixed(2)} per 1,000`)
|
||||
|
||||
// Analyze the nature of clipped districts — are they sparse or not?
|
||||
const extreme = sorted.filter((r) => r.maxX > 30).sort((a, b) => a.totalEnroll - b.totalEnroll)
|
||||
console.log(`\n--- Clipped district enrollment distribution ---`)
|
||||
for (const p of [10, 25, 50, 75, 90]) {
|
||||
const idx = Math.floor((p / 100) * (extreme.length - 1))
|
||||
console.log(` ${p}th percentile enrollment: ${extreme[idx].totalEnroll.toLocaleString()}`)
|
||||
}
|
||||
|
||||
// How many clipped districts have "normal" school sizes (>1000 students)?
|
||||
const normalClipped = extreme.filter((r) => r.totalEnroll >= 1000).length
|
||||
console.log(`\n- ${normalClipped} of ${extreme.length} clipped districts have enrollment ≥ 1,000 (${(normalClipped / extreme.length * 100).toFixed(1)}%)`)
|
||||
|
||||
// Analyze what's driving the extremes — AC bounds vs draw estimates
|
||||
console.log('\n--- What drives extreme values? ---')
|
||||
const acDriven = sorted.filter((r) => r.maxX > 30 && !r.pooled).length
|
||||
console.log(`- Unpooled districts clipped: ${acDriven} (${(acDriven / n * 100).toFixed(2)}% of all)`)
|
||||
|
||||
function percentile(arr, p) {
|
||||
const s = [...arr].sort((a, b) => a - b)
|
||||
return s[Math.floor((p / 100) * (s.length - 1))]
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error)
|
||||
+17
-2
@@ -10,15 +10,22 @@ import { quantile } from './kde.js'
|
||||
import { niceTicks } from './niceTicks.js'
|
||||
|
||||
/**
|
||||
* Hard ceiling on the axis, inherited from the chart this replaces.
|
||||
* Hard ceiling on the axis.
|
||||
*
|
||||
* A four-student cell can produce posterior draws in the hundreds per 1,000;
|
||||
* letting those set the axis squashes every other group into the leftmost few
|
||||
* pixels. `computeRateDomain` reports `clipped` whenever the cap actually
|
||||
* binds, so the chart can say so rather than cropping data off the edge in
|
||||
* silence.
|
||||
*
|
||||
* Set to 100 per 1,000 (one arrest per ten students). The previous value of
|
||||
* 30 was exceeded by over 60% of districts — mostly sparse groups with small
|
||||
* enrollment cells whose Agresti–Coull upper bounds genuinely extend that far.
|
||||
* A cap of 100 still guards against the most degenerate cases (e.g. a single
|
||||
* predicted arrest in a four-student cell yielding ~250 per 1,000) while
|
||||
* letting realistic data drive the axis for the vast majority of districts.
|
||||
*/
|
||||
export const MAX_RATE_DOMAIN = 30
|
||||
export const MAX_RATE_DOMAIN = 100
|
||||
|
||||
/**
|
||||
* Ignore the very top of each draw set when sizing the axis. The posterior
|
||||
@@ -30,6 +37,14 @@ const DOMAIN_QUANTILE = 0.995
|
||||
const HEADROOM = 1.15
|
||||
|
||||
/**
|
||||
* Computes a shared x-domain for the rate density and over-time charts.
|
||||
*
|
||||
* The domain covers everything that will actually be drawn: the 0.995
|
||||
* quantile of each group's posterior-predictive draw rates, plus every
|
||||
* Agresti–Coull upper bound from selected groups. `niceMax` is a clean
|
||||
* "nice number" ceiling (via `niceTicks`) that may exceed the raw data max by
|
||||
* up to one tick step, and is capped at `MAX_RATE_DOMAIN`.
|
||||
*
|
||||
* @param {Array<number[]>} rateArrays - per-group draw rates, per 1,000
|
||||
* @param {number[]} acUpperRates - Agresti–Coull upper bounds, per 1,000
|
||||
* @returns {{ticks: number[], niceMax: number, clipped: boolean}}
|
||||
|
||||
@@ -23,16 +23,27 @@ test('computeRateDomain: one extreme draw does not blow out the axis', () => {
|
||||
assert.ok(domain.niceMax < 10, `niceMax was ${domain.niceMax}`)
|
||||
})
|
||||
|
||||
test('computeRateDomain: caps the domain and reports the clip', () => {
|
||||
test('computeRateDomain: caps at MAX_RATE_DOMAIN=100 and reports the clip', () => {
|
||||
// The cap keeps a tiny-denominator group from flattening every other curve,
|
||||
// but the caller has to be able to say so on screen rather than silently
|
||||
// cropping data off the right edge.
|
||||
const wide = new Array(500).fill(0).map((_, i) => (i < 490 ? 60 : 0))
|
||||
const domain = computeRateDomain([wide], [80])
|
||||
const wide = new Array(500).fill(0).map((_, i) => (i < 490 ? 200 : 0))
|
||||
const domain = computeRateDomain([wide], [300])
|
||||
assert.equal(domain.niceMax, MAX_RATE_DOMAIN)
|
||||
assert.equal(domain.clipped, true)
|
||||
})
|
||||
|
||||
test('computeRateDomain: realistic sparse district data is not clipped', () => {
|
||||
// Carson City NV (LEAID 3200390): pooled BL cell with 34 students and 0
|
||||
// observed arrests → AC upper bound ≈ 88 per 1,000. This should NOT be
|
||||
// clipped under the new cap of 100.
|
||||
const blRates = new Array(500).fill(0)
|
||||
const domain = computeRateDomain([blRates], [88])
|
||||
assert.ok(domain.niceMax >= 88, `niceMax ${domain.niceMax} should cover AC bound of 88`)
|
||||
assert.equal(domain.clipped, false)
|
||||
})
|
||||
|
||||
|
||||
test('computeRateDomain: never returns a zero-width domain', () => {
|
||||
const domain = computeRateDomain([[0, 0, 0]], [0])
|
||||
assert.ok(domain.niceMax > 0)
|
||||
@@ -49,3 +60,14 @@ test('computeRateDomain: ignores empty rate arrays', () => {
|
||||
const domain = computeRateDomain([[], [1, 2, 3]], [])
|
||||
assert.ok(domain.niceMax > 0 && domain.niceMax < MAX_RATE_DOMAIN)
|
||||
})
|
||||
|
||||
test('computeRateDomain: degenerate tiny-denominator cell is clipped', () => {
|
||||
// A four-student cell with a single predicted arrest yields ~250 per
|
||||
// 1,000 — this should be capped at MAX_RATE_DOMAIN=100 and flagged as
|
||||
// clipped so the chart can warn the reader.
|
||||
const tinyRates = new Array(500).fill(0)
|
||||
for (let i = 0; i < 250; i++) tinyRates[i] = 250
|
||||
const domain = computeRateDomain([tinyRates], [300])
|
||||
assert.equal(domain.niceMax, MAX_RATE_DOMAIN)
|
||||
assert.equal(domain.clipped, true)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user