|
| 1 | +/** |
| 2 | + * Histogram bars: the `histogram` summary stat the viewer renders through the |
| 3 | + * `displayer: 'histogram'` pinned row. |
| 4 | + * |
| 5 | + * This is a faithful TS port of buckaroo's Python producer |
| 6 | + * (`customizations/histogram.py`) so a DuckDB-backed column renders the same |
| 7 | + * bars as the pandas/polars backends: |
| 8 | + * |
| 9 | + * - NUMERIC: a low-tail marker, 10 "meat" buckets over the 1st–99th |
| 10 | + * percentile range (population %), a high-tail marker, then an NA bucket. |
| 11 | + * - CATEGORICAL: the top-7 categories (`cat_pop` %), then `longtail`, |
| 12 | + * `unique`, and `NA` buckets. |
| 13 | + * |
| 14 | + * All percentages are on the 0–100 scale and rounded with numpy's |
| 15 | + * round-half-to-even (`np.round`) so the labels match the Python output |
| 16 | + * bit-for-bit. The SQL that produces the raw inputs lives in `histogramSql.ts`. |
| 17 | + */ |
| 18 | + |
| 19 | +import type { HistogramBar } from './wireTypes.js'; |
| 20 | + |
| 21 | +/** numpy `np.round(x, 0)` — round half to even (banker's rounding). */ |
| 22 | +export function npRound0(x: number): number { |
| 23 | + const floor = Math.floor(x); |
| 24 | + const diff = x - floor; |
| 25 | + if (diff < 0.5) return floor; |
| 26 | + if (diff > 0.5) return floor + 1; |
| 27 | + // exact .5 → round to the even neighbour |
| 28 | + return floor % 2 === 0 ? floor : floor + 1; |
| 29 | +} |
| 30 | + |
| 31 | +/** histogram.py:_trim — strip trailing zeros after the decimal point only. */ |
| 32 | +function trim(s: string): string { |
| 33 | + if (s.includes('.')) { |
| 34 | + return s.replace(/0+$/, '').replace(/\.$/, ''); |
| 35 | + } |
| 36 | + return s; |
| 37 | +} |
| 38 | + |
| 39 | +/** histogram.py:fmt_num — SI prefix (K/M/B/T) + step-based precision. */ |
| 40 | +export function fmtNum(value: number, step: number, ref: number): string { |
| 41 | + if (step > 0 && Math.abs(value) < step * 1e-9) { |
| 42 | + value = 0.0; |
| 43 | + } |
| 44 | + const tiers: Array<[number, string]> = [ |
| 45 | + [1e12, 'T'], |
| 46 | + [1e9, 'B'], |
| 47 | + [1e6, 'M'], |
| 48 | + [1e3, 'K'], |
| 49 | + ]; |
| 50 | + for (const [threshold, suffix] of tiers) { |
| 51 | + if (ref >= threshold) { |
| 52 | + const scaled = value / threshold; |
| 53 | + const stepS = step / threshold; |
| 54 | + let dec = stepS > 0 ? Math.max(0, -Math.floor(Math.log10(stepS)) + 1) : 1; |
| 55 | + dec = Math.min(dec, 2); |
| 56 | + return trim(scaled.toFixed(dec)) + suffix; |
| 57 | + } |
| 58 | + } |
| 59 | + let dec = step > 0 ? Math.max(0, -Math.floor(Math.log10(step)) + 1) : 0; |
| 60 | + dec = Math.min(dec, 6); |
| 61 | + return trim(value.toFixed(dec)); |
| 62 | +} |
| 63 | + |
| 64 | +/** |
| 65 | + * histogram.py:_join_bounds — any negative bound switches the separator to |
| 66 | + * `<>`; the minus sign and en-dash are near-identical glyphs and `-0.5–0.5` |
| 67 | + * reads as a double dash. |
| 68 | + */ |
| 69 | +function joinBounds(loS: string, hiS: string): string { |
| 70 | + const sep = loS.startsWith('-') || hiS.startsWith('-') ? '<>' : '–'; |
| 71 | + return `${loS}${sep}${hiS}`; |
| 72 | +} |
| 73 | + |
| 74 | +/** histogram.py:fmt_bucket */ |
| 75 | +export function fmtBucket(lo: number, hi: number, step: number, ref: number): string { |
| 76 | + return joinBounds(fmtNum(lo, step, ref), fmtNum(hi, step, ref)); |
| 77 | +} |
| 78 | + |
| 79 | +/** |
| 80 | + * histogram.py:fmt_tail_bucket — per-bound SI prefix (ref = |bound|) so a far |
| 81 | + * outlier bound doesn't drag a small bound down to '0K'. |
| 82 | + */ |
| 83 | +function fmtTailBucket(lo: number, hi: number, step: number): string { |
| 84 | + return joinBounds(fmtNum(lo, step, Math.abs(lo)), fmtNum(hi, step, Math.abs(hi))); |
| 85 | +} |
| 86 | + |
| 87 | +/** histogram.py:numeric_histogram_labels — one label per bucket from the edges. */ |
| 88 | +export function numericHistogramLabels(endpoints: number[]): string[] { |
| 89 | + let left = endpoints[0]; |
| 90 | + const labels: string[] = []; |
| 91 | + const minVal = endpoints[0]; |
| 92 | + const maxVal = endpoints[endpoints.length - 1]; |
| 93 | + const step = (maxVal - minVal) / Math.max(endpoints.length - 1, 1); |
| 94 | + const ref = Math.max(Math.abs(minVal), Math.abs(maxVal)); |
| 95 | + for (const edge of endpoints.slice(1)) { |
| 96 | + labels.push(fmtBucket(left, edge, step, ref)); |
| 97 | + left = edge; |
| 98 | + } |
| 99 | + return labels; |
| 100 | +} |
| 101 | + |
| 102 | +/** |
| 103 | + * The raw numeric-histogram inputs, mirroring Python's `histogram_args`: |
| 104 | + * - `meatCounts` — np.histogram bin counts (10 bins). |
| 105 | + * - `endpoints` — the 11 bin edges (linspace over the meat range). |
| 106 | + * - `lowTail`/`highTail`— the 1st/99th percentile cut points. |
| 107 | + * `normalizedPopulations` is derived here (counts / sum) so callers only carry |
| 108 | + * the SQL-shaped values. |
| 109 | + */ |
| 110 | +export interface NumericHistogramArgs { |
| 111 | + meatCounts: number[]; |
| 112 | + endpoints: number[]; |
| 113 | + lowTail: number; |
| 114 | + highTail: number; |
| 115 | +} |
| 116 | + |
| 117 | +/** histogram.py:numeric_histogram */ |
| 118 | +export function numericHistogram( |
| 119 | + args: NumericHistogramArgs, |
| 120 | + min: number, |
| 121 | + max: number, |
| 122 | + nanPer: number, |
| 123 | +): HistogramBar[] { |
| 124 | + const naObs: HistogramBar = { name: 'NA', NA: npRound0(nanPer * 100) }; |
| 125 | + if (nanPer === 1.0) return [naObs]; |
| 126 | + |
| 127 | + const { meatCounts, endpoints, lowTail, highTail } = args; |
| 128 | + const total = meatCounts.reduce((a, b) => a + b, 0); |
| 129 | + const normalizedPop = meatCounts.map((c) => (total > 0 ? c / total : 0)); |
| 130 | + const labels = numericHistogramLabels(endpoints); |
| 131 | + |
| 132 | + const eLo = endpoints[0]; |
| 133 | + const eHi = endpoints[endpoints.length - 1]; |
| 134 | + const step = (eHi - eLo) / Math.max(endpoints.length - 1, 1); |
| 135 | + |
| 136 | + const histo: HistogramBar[] = []; |
| 137 | + histo.push({ name: fmtTailBucket(min, lowTail, step), tail: 1 }); |
| 138 | + labels.forEach((label, i) => { |
| 139 | + histo.push({ name: label, population: npRound0(normalizedPop[i] * 100) }); |
| 140 | + }); |
| 141 | + histo.push({ name: fmtTailBucket(highTail, max, step), tail: 1 }); |
| 142 | + if (nanPer > 0.0) histo.push(naObs); |
| 143 | + return histo; |
| 144 | +} |
| 145 | + |
| 146 | +/** A distinct value and its row count, as returned by the value-counts query. */ |
| 147 | +export interface ValueCount { |
| 148 | + name: string; |
| 149 | + count: number; |
| 150 | +} |
| 151 | + |
| 152 | +/** |
| 153 | + * histogram.py:categorical_histogram (with categorical_dict inlined). |
| 154 | + * |
| 155 | + * `top` is the top-N value counts (already sorted desc); `restSum` is the |
| 156 | + * summed count of every other distinct value; `uniqueCount` is the number of |
| 157 | + * distinct values that occur exactly once (over the whole column). `length` |
| 158 | + * includes nulls. |
| 159 | + */ |
| 160 | +export function categoricalHistogram( |
| 161 | + length: number, |
| 162 | + top: ValueCount[], |
| 163 | + restSum: number, |
| 164 | + uniqueCount: number, |
| 165 | + nanPer: number, |
| 166 | +): HistogramBar[] { |
| 167 | + const histo: HistogramBar[] = []; |
| 168 | + for (const { name, count } of top) { |
| 169 | + const percent = npRound0((count / length) * 100); |
| 170 | + if (percent > 0.3) { |
| 171 | + histo.push({ name: String(name), cat_pop: percent }); |
| 172 | + } |
| 173 | + } |
| 174 | + const longTail = restSum - uniqueCount; |
| 175 | + if (longTail > 0) { |
| 176 | + histo.push({ name: 'longtail', longtail: npRound0((longTail / length) * 100) }); |
| 177 | + } |
| 178 | + if (uniqueCount > 0) { |
| 179 | + histo.push({ name: 'unique', unique: npRound0((uniqueCount / length) * 100) }); |
| 180 | + } |
| 181 | + if (nanPer > 0.0) { |
| 182 | + histo.push({ name: 'NA', NA: npRound0(nanPer * 100) }); |
| 183 | + } |
| 184 | + return histo; |
| 185 | +} |
| 186 | + |
| 187 | +/** The categorical inputs `buildHistogram` needs (see `parseCategorical`). */ |
| 188 | +export interface CategoricalArgs { |
| 189 | + top: ValueCount[]; |
| 190 | + restSum: number; |
| 191 | + uniqueCount: number; |
| 192 | +} |
| 193 | + |
| 194 | +/** |
| 195 | + * histogram.py:histogram — pick the numeric path only when the column is |
| 196 | + * numeric, has more than 5 distinct values, valid histogram args, and the |
| 197 | + * resulting numeric histogram has more than 5 bars; otherwise fall back to the |
| 198 | + * categorical histogram. |
| 199 | + * |
| 200 | + * The numeric and categorical inputs are fetched lazily so the categorical |
| 201 | + * query only runs when the numeric path doesn't win — matching histogram.py's |
| 202 | + * short-circuit. This is the single dispatcher: the production pipeline |
| 203 | + * (histogramSql.ts:computeColumnHistogram) supplies SQL-backed fetchers, the |
| 204 | + * unit tests supply in-memory ones, so the same branch logic is what ships. |
| 205 | + */ |
| 206 | +export async function buildHistogram(opts: { |
| 207 | + isNumeric: boolean; |
| 208 | + distinctCount: number; |
| 209 | + length: number; |
| 210 | + nanPer: number; |
| 211 | + min: number | null; |
| 212 | + max: number | null; |
| 213 | + fetchNumericArgs: () => Promise<NumericHistogramArgs | null>; |
| 214 | + fetchCategorical: () => Promise<CategoricalArgs>; |
| 215 | +}): Promise<HistogramBar[]> { |
| 216 | + const { isNumeric, distinctCount, length, nanPer, min, max, fetchNumericArgs, fetchCategorical } = |
| 217 | + opts; |
| 218 | + if (isNumeric && distinctCount > 5 && min !== null && max !== null) { |
| 219 | + const numericArgs = await fetchNumericArgs(); |
| 220 | + if (numericArgs) { |
| 221 | + const temp = numericHistogram(numericArgs, min, max, nanPer); |
| 222 | + if (temp.length > 5) return temp; |
| 223 | + } |
| 224 | + } |
| 225 | + const categorical = await fetchCategorical(); |
| 226 | + return categoricalHistogram(length, categorical.top, categorical.restSum, categorical.uniqueCount, nanPer); |
| 227 | +} |
0 commit comments