Add D3 and new ridgeline chart matching R aesthetic

- Install d3@7 - Add fetchDistrictDraws API function (for future use with raw draws) - Create RateDensityRidgeline component using D3.js - Density ridges showing posterior distributions per race group - Diamond markers for observed rates (matching R design) - Dropdown to switch between model specifications - Matches Civilytics color palette (navy fill, danger diamonds)
2026-08-10 16:15:33 -04:00
parent c4c0aa63bd
commit f65a70fc3e
5 changed files with 797 additions and 4 deletions
@@ -0,0 +1,317 @@
+import { useState, useEffect, useRef } from 'react'
+import * as d3 from 'd3'
+
+const MODEL_LABELS = {
+  unified_m1_mod: 'One-year, baseline',
+  unified_m2_mod: 'One-year + covariate', 
+  unified_m3_mod: 'Three-year, baseline',
+  unified_m4_mod: 'Three-year + covariate',
+}
+
+const RACE_LABELS = {
+  WH: 'White',
+  BL: 'Black', 
+  HI: 'Hispanic',
+  AM: 'American Indian/\nAlaska Native'
+}
+
+export default function RateDensityRidgeline({ quadData, rateByGroup }) {
+  const [selectedModel, setSelectedModel] = useState('unified_m2_mod')
+  const svgRef = useRef(null)
+
+  // Colors matching Civilytics palette
+  const colors = {
+    navy: '#000a9b',
+    teal: '#0791b6', 
+    danger: '#c92d0e',
+    ink: '#222222'
+  }
+
+  useEffect(() => {
+    if (!quadData || !selectedModel) return
+
+    const rows = quadData[selectedModel] || []
+    
+    // Get unique groups for this model
+    const groups = [...new Map(rows.map(r => [`${r.race}-${r.sex}`, { 
+      race: r.race, 
+      sex: r.sex, 
+      label: `${RACE_LABELS[r.race] || r.race} ${r.sex === 'F' ? 'Female' : 'Male'}`
+    }])).values()]
+
+    // Build data for ridgeline
+    const plotData = groups.map(g => {
+      const row = rows.find(r => r.race === g.race && r.sex === g.sex)
+      if (!row) return null
+      
+      // Generate histogram from draws or use rate bounds to approximate
+      const enroll = row.stu_enroll || 1
+      
+      // Create density-like data from the interval
+      const rateMedian = (row.rate_median || 0) * 1000
+      const rateLower = (row.rate_lower || 0) * 1000
+      const rateUpper = (row.rate_upper || 0) * 1000
+      
+      // Generate synthetic histogram from the interval (normal approximation)
+      const sd = (rateUpper - rateLower) / 3.29 // 90% interval
+      const draws = Array.from({ length: 100 }, () => 
+        Math.max(0, d3.randomNormal(rateMedian, sd || 0.1)())
+      )
+      
+      // Observed rate
+      const observedRate = (row.observed_arrests || 0) / ((enroll || 1) / 1000)
+      
+      return {
+        ...g,
+        draws: draws.sort(d3.ascending),
+        observedRate,
+        rateMedian
+      }
+    }).filter(Boolean)
+
+    // Observed rates from rateByGroup for comparison (single model only)
+    const observedData = groups.map(g => {
+      const groupData = rateByGroup.find(r => r.race === g.race && r.sex === g.sex)
+      return groupData ? { ...g, observedRate: groupData.observedRate } : null
+    }).filter(Boolean)
+
+    // Set up dimensions
+    const container = svgRef.current.parentElement
+    const width = Math.min(500, container?.clientWidth || 500)
+    const height = Math.max(300, groups.length * 50 + 80)
+    const margin = { top: 40, right: 30, bottom: 60, left: 80 }
+    const innerWidth = width - margin.left - margin.right
+    const innerHeight = height - margin.top - margin.bottom
+
+    // Clear previous
+    d3.select(svgRef.current).selectAll('*').remove()
+
+    const svg = d3.select(svgRef.current)
+      .attr('width', width)
+      .attr('height', height)
+      .attr('viewBox', `0 0 ${width} ${height}`)
+
+    // Create scales
+    const allRates = plotData.flatMap(d => d.draws).concat(observedData.map(d => d.observedRate))
+    const xMax = Math.min(Math.max(...allRates, 0), 20) || 5 // Cap at 20 per 1k for readability
+    
+    const xScale = d3.scaleLinear()
+      .domain([0, xMax])
+      .range([margin.left, margin.left + innerWidth])
+
+    const yScale = d3.scaleBand()
+      .domain(plotData.map(d => d.label))
+      .range([margin.top, margin.top + innerHeight])
+      .padding(0.3)
+
+    // X-axis
+    svg.append('g')
+      .attr('transform', `translate(0,${margin.top + innerHeight})`)
+      .call(d3.axisBottom(xScale).ticks(10))
+      .call(g => g.select('.domain').attr('stroke', '#ddd'))
+      .call(g => g.selectAll('.tick line').attr('stroke', '#eee'))
+      .call(g => g.selectAll('.tick text').attr('fill', '#666'))
+
+    // X-axis label
+    svg.append('text')
+      .attr('x', margin.left + innerWidth / 2)
+      .attr('y', height - 10)
+      .attr('text-anchor', 'middle')
+      .attr('font-size', '0.75rem')
+      .attr('fill', '#666')
+      .text('Predicted arrests per 1,000 students')
+
+    // Grid lines
+    svg.append('g')
+      .attr('class', 'grid')
+      .selectAll('line')
+      .data(xScale.ticks(10))
+      .join('line')
+      .attr('x1', d => xScale(d))
+      .attr('x2', d => xScale(d))
+      .attr('y1', margin.top)
+      .attr('y2', margin.top + innerHeight)
+      .attr('stroke', '#f0f0f0')
+      .attr('stroke-width', 1)
+
+    // Generate density polygons for each group
+    plotData.forEach((d, i) => {
+      const yPos = yScale(d.label)
+      const rowHeight = yScale.bandwidth()
+      
+      if (!yPos) return
+
+      // Create histogram bins
+      const binGenerator = d3.bin()
+        .domain(xScale.domain())
+        .thresholds(30)
+      
+      const bins = binGenerator(d.draws)
+      const maxCount = d3.max(bins, b => b.length)
+      
+      // Area generator
+      const area = d3.area()
+        .x(b => xScale((b.x0 + b.x1) / 2))
+        .y0(yPos)
+        .y1(yPos)
+        .curve(d3.curveBasis)
+
+      // Scale heights
+      const heightScale = d3.scaleLinear()
+        .domain([0, maxCount])
+        .range([rowHeight * 0.1, rowHeight * 0.9])
+
+      // Draw the ridgeline
+      svg.append('path')
+        .datum(bins)
+        .attr('fill', colors.navy)
+        .attr('opacity', 0.6)
+        .attr('stroke', 'white')
+        .attr('stroke-width', 0.5)
+        .attr('d', d => area.bisector ? 
+          d3.area()
+            .x(b => xScale((b.x0 + b.x1) / 2))
+            .y0(yPos)
+            .y1(b => yPos + heightScale(b.length))
+            .curve(d3.curveBasis)(d)
+          : null
+        )
+        
+      // Fix: manual polygon construction for each bin
+      const points = []
+      bins.forEach(b => {
+        if (b.length > 0) {
+          const xMid = (b.x0 + b.x1) / 2
+          points.push([xScale(xMid), yPos])
+          points.push([xScale(xMid), yPos + heightScale(b.length)])
+        }
+      })
+      
+      // Add top edge
+      for (let i = bins.length - 1; i >= 0; i--) {
+        const b = bins[i]
+        if (b.length > 0) {
+          const xMid = (b.x0 + b.x1) / 2
+          points.push([xScale(xMid), yPos])
+        }
+      }
+
+      if (points.length > 0) {
+        svg.append('path')
+          .attr('d', `M${points.map(p => p.join(',')).join('L')}Z`)
+          .attr('fill', colors.navy)
+          .attr('opacity', 0.55)
+          .attr('stroke', 'white')
+          .attr('stroke-width', 0.5)
+      }
+
+      // Observed rate marker (diamond shape)
+      if (d.observedRate > 0 && d.observedRate <= xMax) {
+        const obsY = yPos + rowHeight / 2
+        
+        // Diamond polygon
+        svg.append('path')
+          .attr('d', `M${xScale(d.observedRate)},${obsY - 6} L${xScale(d.observedRate) + 5},${obsY} L${xScale(d.observedRate)},${obsY + 6} L${xScale(d.observedRate) - 5},${obsY} Z`)
+          .attr('fill', colors.danger)
+          
+        // Label for observed rate
+        svg.append('text')
+          .attr('x', xScale(d.observedRate))
+          .attr('y', obsY - 10)
+          .attr('text-anchor', 'middle')
+          .attr('font-size', '0.65rem')
+          .attr('fill', colors.danger)
+          .attr('font-weight', 600)
+          .text(d.observedRate.toFixed(1))
+      }
+    })
+
+    // Y-axis labels
+    svg.append('g')
+      .attr('transform', `translate(${margin.left},0)`)
+      .call(d3.axisLeft(yScale).tickSize(0))
+      .call(g => g.select('.domain').remove())
+      .call(g => g.selectAll('.tick text')
+        .attr('font-size', '0.75rem')
+        .attr('fill', colors.ink)
+        .attr('font-weight', 500))
+      
+    // Adjust y-axis label text (handle newlines)
+    svg.selectAll('.tick text')
+      .text(d => d.replace('/\n', ' '))
+
+    // Legend
+    const legend = svg.append('g')
+      .attr('transform', `translate(${margin.left + innerWidth - 120}, ${height - 35})`)
+
+    // Diamond legend
+    legend.append('path')
+      .attr('d', 'M0,-4 L3,0 L0,4 L-3,0 Z')
+      .attr('fill', colors.danger)
+    legend.append('text')
+      .attr('x', 8)
+      .attr('y', 2)
+      .attr('font-size', '0.65rem')
+      .attr('fill', '#666')
+      .text('Observed rate')
+
+    // Ridge legend  
+    legend.append('rect')
+      .attr('x', 70)
+      .attr('y', -8)
+      .attr('width', 12)
+      .attr('height', 8)
+      .attr('fill', colors.navy)
+      .attr('opacity', 0.55)
+    legend.append('text')
+      .attr('x', 86)
+      .attr('y', -2)
+      .attr('font-size', '0.65rem')
+      .attr('fill', '#666')
+      .text('Modeled (90%)')
+
+  }, [quadData, selectedModel, rateByGroup])
+
+  if (!quadData || Object.keys(quadData).length === 0) {
+    return (
+      <div className="cv-card" style={{ padding: 'var(--space-2)' }}>
+        <h3 style={{ fontSize: '0.85rem', marginBottom: 'var(--space-1)', color: 'var(--cv-ink-2)' }}>
+          Predicted arrest rates by student group
+        </h3>
+        <p style={{ color: 'var(--cv-ink-3)' }}>No model data available.</p>
+      </div>
+    )
+  }
+
+  return (
+    <div className="cv-card" style={{ padding: 'var(--space-2)' }}>
+      {/* Chart title */}
+      <h3 style={{ fontSize: '0.85rem', marginBottom: 'var(--space-1)', color: 'var(--cv-ink-2)' }}>
+        Predicted arrest rates by student group
+      </h3>
+
+      {/* Model selector */}
+      <select 
+        value={selectedModel} 
+        onChange={(e) => setSelectedModel(e.target.value)}
+        style={{ marginBottom: 'var(--space-2)', padding: '0.25rem 0.5rem', fontFamily: 'var(--font-sans)' }}
+      >
+        {Object.entries(MODEL_LABELS).map(([val, label]) => (
+          <option key={val} value={val}>{label}</option>
+        ))}
+      </select>
+
+      {/* D3 Ridgeline */}
+      <div style={{ width: '100%', overflowX: 'auto' }}>
+        <svg ref={svgRef} style={{ width: '100%', minWidth: 400, height: 'auto' }} />
+      </div>
+
+      {/* Note */}
+      <p style={{ fontSize: '0.72rem', color: 'var(--cv-ink-3)', marginTop: 'var(--space-1)' }}>
+        <strong>Density ridges</strong> show the posterior distribution of predicted arrests per 1,000 students. 
+        <strong>Diamonds</strong> mark the observed rate for each group (the raw data point the model aims to improve).
+        Using 90% intervals from the {MODEL_LABELS[selectedModel]} model.
+      </p>
+    </div>
+  )
+}
@@ -4,7 +4,7 @@ import ArrestsOverTime from '../charts/ArrestsOverTime.jsx'
 import RateByGroupBar from '../charts/RateByGroupBar.jsx'
 import DistrictVsNational from '../charts/DistrictVsNational.jsx'
 import ModelDrawsComparison from '../charts/ModelDrawsComparison.jsx'
-import ObservedRateDensity from '../charts/ObservedRateDensity.jsx'
+import RateDensityRidgeline from '../charts/RateDensityRidgeline.jsx'
 import ExceedanceProbability from '../charts/ExceedanceProbability.jsx'

 /**
@@ -118,7 +118,7 @@ export default function ChartPanel({ district, state }) {
        {/* Charts 4–6: Bayesian model distributions */}
        <ModelDrawsComparison data={{ ...chart4Data, quadData: chart5Data }} waves={['15-16', '17-18', '21-22']} districtName={district.lea_name} />

-        <ObservedRateDensity quadData={chart5Data} />
+        <RateDensityRidgeline quadData={chart5Data} rateByGroup={rateByGroup} />

        <ExceedanceProbability rateByGroup={rateByGroup} nationalRates={nationalRates?.rates_per_1000 || {}} />
      </div>
@@ -134,3 +134,15 @@ export async function fetchDrawLocation(state, race, sex, year, model) {

  return apiFetch(`/draws?${params}`)
 }
+
+/** GET /estimates/{leaid}/draws?model=X&year=Y — fetch posterior draws for a district */
+export async function fetchDistrictDraws(leaid, options = {}) {
+  const params = new URLSearchParams()
+  if (options.model) params.set('model', options.model)
+  if (options.year) params.set('year', options.year)
+  if (options.race) params.set('race', options.race)
+  if (options.sex) params.set('sex', options.sex)
+
+  const qs = params.toString()
+  return apiFetch(`/estimates/${leaid}/draws${qs ? `?${qs}` : ''}`)
+}