Redesign chart visualizations to match Civilytics white-paper style

Rebuilds all 6 charts around a validated categorical race palette, row-based
sex encoding, and a consistent observed-vs-modeled mark convention (diamond
vs. filled bar/density) instead of ad hoc per-chart color schemes. Adds a
shared student-group filter (defaults to all 8 groups) that scopes every
chart's data from one place in ChartPanel.

Drops the D3 dependency entirely in favor of plain SVG, removing the
imperative-DOM bug class behind this app's repeated "fix the fix" commits.
Replaces the fake symmetric-normal posterior approximation with a skewed,
median-preserving fit to the API's interval bounds, clearly labeled as an
approximation. Fixes two broken SVG fill attributes, a decorative model
dropdown that never affected its chart, dead code, an orphaned component,
and a broken CSS token reference.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/charts/RateDensityRidgeline.jsx

@@ -1,270 +1,39 @@
-import { useState, useEffect, useRef } from 'react'
-import * as d3 from 'd3'
+import { MODEL_QUADRANTS } from '../hooks/useApi.js'
+import { raceColor, OBSERVED_MARK_COLOR, RACE_LABELS } from '../utils/colors.js'
+import { fitSkewedInterval, densityCurve } from '../utils/distributionApprox.js'
+import ChartLegend from '../components/ChartLegend.jsx'
+import ApproxNote from '../components/ApproxNote.jsx'
 
-const MODEL_LABELS = {
-  unified_m1_mod: 'One-year, baseline',
-  unified_m2_mod: 'One-year + covariate',
-  unified_m3_mod: 'Three-year, baseline',
-  unified_m4_mod: 'Three-year + covariate',
+/**
+ * Chart 5: Modeled posterior density per race×sex group, faceted 2×2 by
+ * model quadrant (matches whitepaper-fig-clark-density-1.png). Plain SVG —
+ * no D3, no fake normal draws: each ridge is drawn as 60 analytic points
+ * from fitSkewedInterval/densityCurve, which is already smooth without a
+ * binning/curveBasis smoothing pass.
+ */
+
+const RACE_ORDER = ['WH', 'BL', 'HI', 'AM']
+const SEX_ORDER = ['F', 'M']
+const SHORT_RACE_LABEL = { WH: 'White', BL: 'Black', HI: 'Hispanic', AM: 'AI/AN' }
+
+function buildGroupRow(row) {
+  const enroll = row.stu_enroll || 0
+  const observedRate = enroll > 0 ? ((row.observed_arrests || 0) / enroll) * 1000 : 0
+  const rateMedian = (row.rate_median || 0) * 1000
+  const rateLower = (row.rate_lower || 0) * 1000
+  const rateUpper = (row.rate_upper || 0) * 1000
+  return {
+    race: row.race,
+    sex: row.sex,
+    observedRate,
+    fit: fitSkewedInterval({ median: rateMedian, lower: rateLower, upper: rateUpper }),
+  }
 }
 
-const RACE_LABELS = {
-  WH: 'White',
-  BL: 'Black',
-  HI: 'Hispanic',
-  AM: 'American Indian / Alaska Native'
-}
+export default function RateDensityRidgeline({ quadData }) {
+  const hasData = quadData && Object.values(quadData).some((rows) => (rows || []).length > 0)
 
-export default function RateDensityRidgeline({ quadData, rateByGroup }) {
-  const [selectedModel, setSelectedModel] = useState('unified_m2_mod')
-  const svgRef = useRef(null)
-
-  useEffect(() => {
-    if (!quadData || !selectedModel || !quadData[selectedModel]) return
-
-    const rows = quadData[selectedModel] || []
-
-    // Get unique groups for this model (race × sex)
-    const groupKeys = [...new Set(rows.map(r => `${r.race}-${r.sex}`))]
-
-    // Build data: for each group, estimate density from the interval
-    const plotData = groupKeys.map(key => {
-      const row = rows.find(r => `${r.race}-${r.sex}` === key)
-      if (!row) return null
-
-      const race = row.race || 'WH'
-      const sex = row.sex || 'F'
-      const label = `${RACE_LABELS[race] || race} ${sex === 'F' ? 'Female' : 'Male'}`
-
-      // Convert rate estimates to per-1000 scale
-      const enroll = row.stu_enroll || 1
-      const obsArrests = row.observed_arrests || 0
-      const observedRate = enroll > 0 ? (obsArrests / enroll) * 1000 : 0
-
-      // Model rate estimates in per-1k units
-      const rateMedian = ((row.rate_median || 0) * 1000) / (enroll > 0 ? 1 : 1)
-      const rateLower = ((row.rate_lower || 0) * 1000) / (enroll > 0 ? 1 : 1)
-      const rateUpper = ((row.rate_upper || 0) * 1000) / (enroll > 0 ? 1 : 1)
-
-      // Generate draws from normal approximation of the posterior interval
-      // Use 90% CI: sd = (upper - lower) / 3.29
-      const intervalWidth = Math.abs(rateUpper - rateLower) || 0.1
-      const sd = intervalWidth / 3.29
-
-      // Generate a larger sample for smoother density estimation
-      const draws = Array.from({ length: 500 }, () =>
-        Math.max(0, d3.randomNormal(rateMedian, sd)() )
-      ).sort(d3.ascending)
-
-      return { key, label, race, sex, rateMedian, observedRate, draws }
-    }).filter(Boolean)
-
-    // Set up dimensions
-    const container = svgRef.current?.parentElement
-    const width = Math.min(500, container?.clientWidth || 500)
-    const height = Math.max(320, plotData.length * 60 + 80)
-    const margin = { top: 40, right: 30, bottom: 70, left: 90 }
-    const innerWidth = width - margin.left - margin.right
-    const innerHeight = height - margin.top - margin.bottom
-
-    // Clear previous
-    d3.select(svgRef.current).selectAll('*').remove()
-
-    const svg = d3.select(svgRef.current)
-      .attr('width', width)
-      .attr('height', height)
-      .attr('viewBox', `0 0 ${width} ${height}`)
-
-    // Compute x-scale domain from all data (draws + observed rates)
-    const allDrawValues = plotData.flatMap(d => d.draws).filter(v => v <= 25)
-    const maxRate = Math.min(Math.max(...allDrawValues, ...plotData.map(d => d.observedRate), 0.1), 25)
-
-    const xScale = d3.scaleLinear()
-      .domain([0, maxRate])
-      .range([margin.left, margin.left + innerWidth])
-
-    // Y-scale: one band per group (race × sex)
-    const yScale = d3.scaleBand()
-      .domain(plotData.map(d => d.label))
-      .range([margin.top, margin.top + innerHeight])
-      .padding(0.25)
-
-    // Title
-    svg.append('text')
-      .attr('x', margin.left + innerWidth / 2)
-      .attr('y', 18)
-      .attr('text-anchor', 'middle')
-      .attr('font-size', '0.7rem')
-      .attr('fill', '#666')
-      .text(`Posterior predicted arrests per 1,000 students — ${MODEL_LABELS[selectedModel]}`)
-
-    // Grid lines
-    svg.append('g')
-      .attr('class', 'grid')
-      .selectAll('line')
-      .data(xScale.ticks(8))
-      .join('line')
-      .attr('x1', d => xScale(d))
-      .attr('x2', d => xScale(d))
-      .attr('y1', margin.top)
-      .attr('y2', margin.top + innerHeight)
-      .attr('stroke', '#f0f0f0')
-      .attr('stroke-width', 1)
-
-    // X-axis
-    svg.append('g')
-      .attr('transform', `translate(0,${margin.top + innerHeight})`)
-      .call(d3.axisBottom(xScale).ticks(8))
-      .call(g => g.select('.domain').attr('stroke', '#ddd'))
-      .call(g => g.selectAll('.tick line').remove())
-      .call(g => g.selectAll('.tick text')
-        .attr('font-size', '0.65rem')
-        .attr('fill', '#888'))
-
-    // X-axis label
-    svg.append('text')
-      .attr('x', margin.left + innerWidth / 2)
-      .attr('y', height - 12)
-      .attr('text-anchor', 'middle')
-      .attr('font-size', '0.7rem')
-      .attr('fill', '#888')
-      .text('Arrests per 1,000 students')
-
-    // Y-axis
-    svg.append('g')
-      .attr('transform', `translate(${margin.left},0)`)
-      .call(d3.axisLeft(yScale).tickSize(0))
-      .call(g => g.select('.domain').remove())
-      .call(g => g.selectAll('.tick text')
-        .attr('font-size', '0.65rem')
-        .attr('fill', '#444'))
-
-    // Colors matching Civilytics palette
-    const fillColor = '#000a9b'  // navy
-    const obsColor = '#c92d0e'   // danger red
-
-    // Draw each ridge (one per race × sex group)
-    plotData.forEach(d => {
-      const yTop = yScale(d.label) || 0
-      const bandwidth = yScale.bandwidth() || 40
-      const baselineY = yTop + bandwidth * 0.15  // bottom of the ridge area
-      const peakHeight = bandwidth * 0.8          // available height for density
-
-      // Create histogram bins from draws, then smooth into a ridge shape
-      const nBins = 40
-      const binWidth = maxRate / nBins
-      const bins = new Array(nBins).fill(0)
-
-      d.draws.forEach(val => {
-        if (val <= maxRate) {
-          const idx = Math.min(Math.floor(val / binWidth), nBins - 1)
-          if (idx >= 0) bins[idx]++
-        }
-      })
-
-      // Find peak count for scaling
-      const maxCount = d3.max(bins) || 1
-      const heightScale = peakHeight * 0.85 / maxCount
-
-      // Build the top edge points of the ridge (smoothed with curveBasis)
-      const topPoints = bins.map((count, i) => {
-        if (count === 0) return null
-        const xMid = margin.left + ((i + 0.5) * binWidth / maxRate) * innerWidth
-        // Ridge grows upward from baseline: higher count → taller ridge (lower y value)
-        const yVal = baselineY - Math.min(count * heightScale, peakHeight)
-        return [xMid, yVal]
-      }).filter(Boolean)
-
-      if (topPoints.length < 2) return
-
-      // Smooth the top edge using basis interpolation
-      const lineGen = d3.line()
-        .curve(d3.curveBasis)
-        .x(d => d[0])
-        .y(d => d[1])
-
-      const smoothedTop = lineGen(topPoints) || ''
-
-      if (!smoothedTop) return
-
-      // Build the complete area path: bottom edge + smoothed top + close
-      // The fill goes from baselineY down to peakHeight (upward in SVG coords, so y decreases)
-      const firstX = margin.left
-      const lastX = margin.left + innerWidth
-
-      let ridgePath = `M${firstX},${baselineY}`           // start at bottom-left of ridge
-      ridgePath += `L${topPoints[0][0]},${baselineY}`    // line to first data x (along baseline)
-      ridgePath += smoothedTop.substring(1)              // append the smoothed top curve (skip 'M')
-      ridgePath += `L${lastX},${baselineY}Z`             // close back along bottom
-
-      svg.append('path')
-        .attr('d', ridgePath)
-        .attr('fill', fillColor)
-        .attr('opacity', 0.55)
-        .attr('stroke', 'white')
-        .attr('stroke-width', 0.5)
-
-      // Observed rate marker — a diamond sitting on top of each ridge
-      if (d.observedRate > 0 && d.observedRate <= maxRate) {
-        const obsX = xScale(d.observedRate)
-        const obsY = baselineY - peakHeight * 0.2  // position above the density
-
-        // Diamond shape (like ggplot2's shape=18)
-        svg.append('polygon')
-          .attr('points', [
-            `${obsX},${obsY - 4}`,
-            `${obsX + 4},${obsY}`,
-            `${obsX},${obsY + 4}`,
-            `${obsX - 4},${obsY}`
-          ].join(' '))
-          .attr('fill', obsColor)
-
-        // Label the observed rate value below the diamond
-        svg.append('text')
-          .attr('x', obsX)
-          .attr('y', yTop + bandwidth * 0.65)
-          .attr('text-anchor', 'middle')
-          .attr('font-size', '0.6rem')
-          .attr('fill', '#888')
-          .text(d.observedRate.toFixed(1))
-      }
-    })
-
-    // Legend
-    const legendY = height - 25
-    svg.append('rect')
-      .attr('x', margin.left)
-      .attr('y', legendY)
-      .attr('width', 12)
-      .attr('height', 8)
-      .attr('fill', fillColor)
-      .attr('opacity', 0.55)
-    svg.append('text')
-      .attr('x', margin.left + 16)
-      .attr('y', legendY + 7)
-      .attr('font-size', '0.62rem')
-      .attr('fill', '#888')
-      .text('Modeled posterior (90% interval)')
-
-    // Observed diamond legend
-    svg.append('polygon')
-      .attr('points', [
-        `${margin.left + 140},${legendY}`,
-        `${margin.left + 144},${legendY + 5}`,
-        `${margin.left + 140},${legendY + 10}`,
-        `${margin.left + 136},${legendY + 5}`
-      ].join(' '))
-      .attr('fill', obsColor)
-    svg.append('text')
-      .attr('x', margin.left + 150)
-      .attr('y', legendY + 7)
-      .attr('font-size', '0.62rem')
-      .attr('fill', '#888')
-      .text('Observed rate (diamond)')
-
-  }, [quadData, selectedModel])
-
-  if (!quadData || Object.keys(quadData).length === 0) {
+  if (!hasData) {
     return (
       <div className="cv-card" style={{ padding: 'var(--space-2)' }}>
         <h3 style={{ fontSize: '0.85rem', marginBottom: 'var(--space-1)', color: 'var(--cv-ink-2)' }}>
@@ -275,34 +44,121 @@ export default function RateDensityRidgeline({ quadData, rateByGroup }) {
     )
   }
 
+  // Shared x-domain across all four quadrant panels, so they're directly comparable.
+  const allUpper = []
+  const allObserved = []
+  MODEL_QUADRANTS.forEach((q) => {
+    (quadData[q.model] || []).forEach((row) => {
+      allUpper.push((row.rate_upper || 0) * 1000)
+      const enroll = row.stu_enroll || 0
+      if (enroll > 0) allObserved.push(((row.observed_arrests || 0) / enroll) * 1000)
+    })
+  })
+  const maxRate = Math.min(Math.max(...allUpper, ...allObserved, 1) * 1.15, 30)
+
   return (
     <div className="cv-card" style={{ padding: 'var(--space-2)' }}>
-      {/* Chart title */}
       <h3 style={{ fontSize: '0.85rem', marginBottom: 'var(--space-1)', color: 'var(--cv-ink-2)' }}>
         Predicted arrest rates by student group
       </h3>
+      <ApproxNote />
 
-      {/* Model selector */}
-      <select
-        value={selectedModel}
-        onChange={(e) => setSelectedModel(e.target.value)}
-        style={{ marginBottom: 'var(--space-2)', padding: '0.25rem 0.5rem', fontFamily: 'var(--font-sans)' }}
-      >
-        {Object.entries(MODEL_LABELS).map(([val, label]) => (
-          <option key={val} value={val}>{label}</option>
+      <div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 'var(--space-2)', marginTop: 'var(--space-1)' }}>
+        {MODEL_QUADRANTS.map((q) => (
+          <RidgePanel key={q.model} q={q} rows={quadData[q.model] || []} maxRate={maxRate} />
         ))}
-      </select>
-
-      {/* D3 Ridgeline */}
-      <div style={{ width: '100%', overflowX: 'auto' }}>
-        <svg ref={svgRef} style={{ width: '100%', minWidth: 420, height: 'auto' }} />
       </div>
 
-      {/* Note */}
-      <p style={{ fontSize: '0.7rem', color: 'var(--cv-ink-3)', marginTop: 'var(--space-1)' }}>
-        <strong>Density ridges</strong> show the posterior distribution of predicted arrests per 1,000 students.
-        <strong>Diamonds</strong> mark observed rates for each group (the raw data point being modeled).
-      </p>
+      <ChartLegend items={[
+        ...RACE_ORDER.map((race) => ({ shape: 'swatch', color: raceColor(race), label: RACE_LABELS[race] })),
+        { shape: 'diamond', color: OBSERVED_MARK_COLOR, label: 'Observed' },
+      ]} />
     </div>
   )
-}
+}
+
+function RidgePanel({ q, rows, maxRate }) {
+  const groups = []
+  SEX_ORDER.forEach((sex) => {
+    RACE_ORDER.forEach((race) => {
+      const row = rows.find((r) => r.race === race && r.sex === sex)
+      if (row) groups.push(buildGroupRow(row))
+    })
+  })
+
+  if (groups.length === 0) {
+    return (
+      <div style={{ border: '1px solid var(--cv-rule)', borderRadius: 'var(--radius-md)', padding: 'var(--space-2)' }}>
+        <p style={{ fontSize: '0.65rem', color: 'var(--cv-ink-3)' }}>{q.label} — no data for the selected groups</p>
+      </div>
+    )
+  }
+
+  const width = 300
+  const rowHeight = 46
+  const margin = { top: 24, right: 16, bottom: 26, left: 56 }
+  const innerWidth = width - margin.left - margin.right
+  const height = margin.top + groups.length * rowHeight + margin.bottom
+
+  const xScale = (val) => margin.left + (val / maxRate) * innerWidth
+
+  const curves = groups.map((g) => densityCurve(g.fit, { min: 0, max: maxRate, n: 60 }))
+  const maxPdf = Math.max(...curves.flatMap((c) => c.map((p) => p.y)), 1e-9)
+  const peakHeight = rowHeight * 0.85
+
+  return (
+    <div style={{ border: '1px solid var(--cv-rule)', borderRadius: 'var(--radius-md)', padding: 'var(--space-1)' }}>
+      <svg width="100%" viewBox={`0 0 ${width} ${height}`} style={{ maxWidth: '100%' }}>
+        <text x={width / 2} y={14} textAnchor="middle" fontSize="0.65rem" fontWeight={600} fill="var(--cv-ink-2)">
+          {q.label}
+        </text>
+
+        {[0, 0.25, 0.5, 0.75, 1].map((frac) => {
+          const val = frac * maxRate
+          const x = xScale(val)
+          return (
+            <g key={frac}>
+              <line x1={x} y1={margin.top} x2={x} y2={margin.top + groups.length * rowHeight} stroke="var(--cv-rule)" strokeWidth={1} />
+              <text x={x} y={margin.top + groups.length * rowHeight + 14} textAnchor="middle" fontSize="0.55rem" fill="var(--cv-ink-3)">
+                {val.toFixed(0)}
+              </text>
+            </g>
+          )
+        })}
+
+        {groups.map((g, i) => {
+          const rowTop = margin.top + i * rowHeight
+          const baselineY = rowTop + rowHeight * 0.9
+          const curve = curves[i]
+          const color = raceColor(g.race)
+
+          const topPath = curve
+            .map((p, j) => `${j === 0 ? 'M' : 'L'}${xScale(p.x)},${baselineY - (p.y / maxPdf) * peakHeight}`)
+            .join(' ')
+          const areaPath = `${topPath} L${xScale(maxRate)},${baselineY} L${xScale(0)},${baselineY} Z`
+
+          const obsX = xScale(Math.min(g.observedRate, maxRate))
+          const obsY = baselineY - peakHeight * 0.15
+
+          return (
+            <g key={`${g.race}-${g.sex}`}>
+              <text x={margin.left - 8} y={rowTop + rowHeight / 2 + 4} textAnchor="end" fontSize="0.6rem" fill="var(--cv-ink)">
+                {SHORT_RACE_LABEL[g.race] || g.race} {g.sex}
+              </text>
+              <path d={areaPath} fill={color} opacity={0.6} stroke="#fff" strokeWidth={0.5} />
+              <rect
+                x={obsX - 4} y={obsY - 4} width={8} height={8}
+                fill={OBSERVED_MARK_COLOR} stroke="#fff" strokeWidth={1}
+                transform={`rotate(45 ${obsX} ${obsY})`}
+              />
+            </g>
+          )
+        })}
+
+        <text x={width / 2} y={height - 6} textAnchor="middle" fontSize="0.55rem" fill="var(--cv-ink-3)">
+          Arrests per 1,000 students
+        </text>
+      </svg>
+    </div>
+  )
+}