import { useEffect, useState } from 'react' import { getDb } from '../utils/duckdbClient.js' const HF_BASE = 'https://huggingface.co/datasets/civilytics/crdc-school-arrest-rates/resolve/main/parquet' // Module-level cache: one registered duckdb-wasm file buffer per // (model, year, state) shard, shared across every component instance and // district navigated to in this browser session. See Global Constraints — // in-memory only, no persistence across page loads. const shardCache = new Map() function shardKey(model, year, state) { return `${model}__${year}__${state}` } function ensureShardRegistered(db, model, year, state) { const key = shardKey(model, year, state) if (!shardCache.has(key)) { shardCache.set( key, (async () => { const url = `${HF_BASE}/model_id=${model}/YEAR=${year}/LEA_STATE=${state}/data_0.parquet` const res = await fetch(url) if (!res.ok) throw new Error(`Failed to fetch draw shard: HTTP ${res.status}`) const buffer = new Uint8Array(await res.arrayBuffer()) const fileName = `${key}.parquet` await db.registerFileBuffer(fileName, buffer) return fileName })(), ) } return shardCache.get(key) } /** * Fetches real posterior draws for one district/model/year from the Hugging * Face parquet dataset via duckdb-wasm, converts predicted counts to * rate-per-1,000 using each group's stu_enroll (not present in the draws * table itself — joined here from data this app already has), and returns * them keyed by "RACE_SEX". * * @param {{leaid: string, state: string, model: string, year: string, * groups: Array<{race: string, sex: string, stuEnroll: number}>}} params * @returns {{status: 'loading'|'ready'|'error', drawsByGroup: Record | null}} */ export function useDrawDistribution({ leaid, state, model, year, groups }) { const [status, setStatus] = useState('loading') const [drawsByGroup, setDrawsByGroup] = useState(null) // groups is typically a fresh array literal every render; derive a stable // primitive so the effect only re-runs when its actual content changes. const groupsSignature = (groups || []).map((g) => `${g.race}:${g.sex}:${g.stuEnroll}`).join(',') useEffect(() => { if (!leaid || !state || !model || !year || !groups?.length) return let cancelled = false setStatus('loading') async function run() { let conn try { const db = await getDb() const fileName = await ensureShardRegistered(db, model, year, state) conn = await db.connect() const stmt = await conn.prepare(`SELECT RACE, SEX, pred FROM read_parquet('${fileName}') WHERE LEAID = ?`) const table = await stmt.query(leaid) await stmt.close() const rows = table.toArray().map((r) => r.toJSON()) const enrollByGroup = {} for (const g of groups) enrollByGroup[`${g.race}_${g.sex}`] = g.stuEnroll || 0 const byGroup = {} for (const row of rows) { const key = `${row.RACE}_${row.SEX}` const enroll = enrollByGroup[key] if (!enroll) continue const rate = (Number(row.pred) / enroll) * 1000 ;(byGroup[key] ??= []).push(rate) } if (!cancelled) { setDrawsByGroup(byGroup) setStatus('ready') } } catch (err) { console.error('useDrawDistribution failed:', err) if (!cancelled) setStatus('error') } finally { if (conn) await conn.close() } } run() return () => { cancelled = true } // eslint-disable-next-line react-hooks/exhaustive-deps }, [leaid, state, model, year, groupsSignature]) return { status, drawsByGroup } }