mirror of
https://github.com/pupperpowell/bibdle.git
synced 2026-10-22 05:54:10 -04:00
first pass analytics redo
This commit is contained in:
1 parent
b656fd04c0
commit
45ffa39f2a
69 files changed
+6400
-1138
No files matched your search
@@ -0,0 +1,31 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { parseAcquisition } from '$lib/utils/acquisition';
|
||||
|
||||
describe('parseAcquisition', () => {
|
||||
test('extracts utm parameters and ref', () => {
|
||||
const a = parseAcquisition('?utm_source=bluesky&utm_medium=social&utm_campaign=launch', '');
|
||||
expect(a.utm_source).toBe('bluesky');
|
||||
expect(a.utm_medium).toBe('social');
|
||||
expect(a.utm_campaign).toBe('launch');
|
||||
});
|
||||
|
||||
test('falls back to ref for the source', () => {
|
||||
expect(parseAcquisition('?ref=hn', '').utm_source).toBe('hn');
|
||||
});
|
||||
|
||||
test('keeps an external referrer', () => {
|
||||
const a = parseAcquisition('', 'https://news.ycombinator.com/item?id=1');
|
||||
expect(a.referrer).toBe('https://news.ycombinator.com/item?id=1');
|
||||
});
|
||||
|
||||
test('blank and whitespace values become null', () => {
|
||||
const a = parseAcquisition('?utm_source=%20%20', '');
|
||||
expect(a.utm_source).toBeNull();
|
||||
expect(a.referrer).toBeNull();
|
||||
});
|
||||
|
||||
test('caps absurdly long values', () => {
|
||||
const a = parseAcquisition(`?utm_source=${'x'.repeat(500)}`, '');
|
||||
expect(a.utm_source!.length).toBe(256);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { composeDigest } from '$lib/server/stats/digest';
|
||||
import type { SnapshotPayload } from '$lib/types/stats';
|
||||
|
||||
const payload: SnapshotPayload = {
|
||||
asOfDate: '2026-09-14',
|
||||
activity: {
|
||||
dau: 102,
|
||||
wau: 363,
|
||||
mau: 852,
|
||||
core: 61,
|
||||
returningShare: 38,
|
||||
newPlayers: 20,
|
||||
resurrected: 4,
|
||||
churned: 25,
|
||||
netGrowth: -1,
|
||||
corePrevious: 60,
|
||||
previous: null
|
||||
},
|
||||
dailySeries: [],
|
||||
kpis: {
|
||||
d1Return: 12.2,
|
||||
return7: 18,
|
||||
overallReturnRate: 22.8,
|
||||
completionRate: null,
|
||||
shareRate: null,
|
||||
firstTryRate: 33.7,
|
||||
avgGuesses: 4.1
|
||||
},
|
||||
retention: { cohorts: [], curves: { d1: [], d7: [], d30: [] }, timeToSecondPlay: [], segmentMix: [] },
|
||||
verseMix: {
|
||||
coverage: [],
|
||||
testamentMix: [],
|
||||
entropy: [],
|
||||
repeatIntervals: [],
|
||||
firstTryByBook: [],
|
||||
retentionByBook: [],
|
||||
retentionByGuessBucket: [],
|
||||
achievability: []
|
||||
},
|
||||
habit: { streakSurvival: [], streakLengths: [], calendar: [], weekdayHour: [] },
|
||||
growth: { lifecycleFunnel: [], signups: [] },
|
||||
totals: { completions: 31775, players: 6589, yesterdayCompletions: 102, yesterdayAvgGuesses: 4.1 },
|
||||
signals: [
|
||||
{
|
||||
id: 'd1_return',
|
||||
severity: 'bad',
|
||||
title: 'D1 return',
|
||||
detail: 'D1 return is 12.2%, below the 20% target.',
|
||||
action: 'Test a "come back tomorrow" prompt.'
|
||||
}
|
||||
],
|
||||
meta: { hasStartedData: false, hasShareData: false, firstEventDate: null }
|
||||
};
|
||||
|
||||
describe('composeDigest', () => {
|
||||
test('includes the core numbers in plain language', () => {
|
||||
const message = composeDigest(payload);
|
||||
expect(message).toContain('2026-09-14');
|
||||
expect(message).toContain('363');
|
||||
expect(message).toContain('12.2');
|
||||
expect(message).toContain('Top signal: D1 return');
|
||||
expect(message).toContain('/admin/stats?date=2026-09-14');
|
||||
});
|
||||
|
||||
test('is at most a handful of lines', () => {
|
||||
expect(composeDigest(payload).split('\n').length).toBeLessThanOrEqual(6);
|
||||
});
|
||||
|
||||
test('shows an em-dash for missing metrics', () => {
|
||||
const message = composeDigest({
|
||||
...payload,
|
||||
kpis: { ...payload.kpis, d1Return: null, return7: null }
|
||||
});
|
||||
expect(message).toContain('D1 return —%');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { eq, and } from 'drizzle-orm';
|
||||
import { testDb } from '$lib/server/db/test';
|
||||
import { analyticsEvents } from '$lib/server/db/schema';
|
||||
import {
|
||||
recordEvent,
|
||||
isEventName,
|
||||
encodeMetadata
|
||||
} from '$lib/server/stats/events';
|
||||
import { MAX_METADATA_BYTES } from '$lib/types/stats';
|
||||
|
||||
async function clear() {
|
||||
await testDb.delete(analyticsEvents);
|
||||
}
|
||||
|
||||
describe('analytics events', () => {
|
||||
beforeEach(clear);
|
||||
afterEach(clear);
|
||||
|
||||
test('allowlist accepts known event names and rejects others', () => {
|
||||
expect(isEventName('verse_served')).toBe(true);
|
||||
expect(isEventName('donate_click')).toBe(true);
|
||||
expect(isEventName('not_a_real_event')).toBe(false);
|
||||
expect(isEventName('')).toBe(false);
|
||||
});
|
||||
|
||||
test('metadata is capped', () => {
|
||||
const small = encodeMetadata({ utm_source: 'bluesky' });
|
||||
expect(small).toBe('{"utm_source":"bluesky"}');
|
||||
const big = encodeMetadata({ blob: 'x'.repeat(MAX_METADATA_BYTES + 100) });
|
||||
expect(big).toBe('{"truncated":true}');
|
||||
expect(encodeMetadata(null)).toBeNull();
|
||||
});
|
||||
|
||||
test('records a normal event', async () => {
|
||||
const inserted = await recordEvent(
|
||||
{ name: 'donate_click', anonymousId: 'anon-1' },
|
||||
testDb
|
||||
);
|
||||
expect(inserted).toBe(true);
|
||||
const rows = await testDb.select().from(analyticsEvents);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].name).toBe('donate_click');
|
||||
});
|
||||
|
||||
test('verse_served is deduped per player + date', async () => {
|
||||
const first = await recordEvent(
|
||||
{ name: 'verse_served', anonymousId: 'anon-1', date: '2026-09-01' },
|
||||
testDb
|
||||
);
|
||||
const second = await recordEvent(
|
||||
{ name: 'verse_served', anonymousId: 'anon-1', date: '2026-09-01' },
|
||||
testDb
|
||||
);
|
||||
expect(first).toBe(true);
|
||||
expect(second).toBe(false);
|
||||
const rows = await testDb
|
||||
.select()
|
||||
.from(analyticsEvents)
|
||||
.where(
|
||||
and(
|
||||
eq(analyticsEvents.name, 'verse_served'),
|
||||
eq(analyticsEvents.anonymousId, 'anon-1')
|
||||
)
|
||||
);
|
||||
expect(rows).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('other event names are not deduped', async () => {
|
||||
await recordEvent({ name: 'share', anonymousId: 'anon-1', date: '2026-09-01' }, testDb);
|
||||
await recordEvent({ name: 'share', anonymousId: 'anon-1', date: '2026-09-01' }, testDb);
|
||||
const rows = await testDb.select().from(analyticsEvents);
|
||||
expect(rows).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('user attribution is stored', async () => {
|
||||
await recordEvent(
|
||||
{ name: 'share_copy', anonymousId: 'anon-1', userId: 'user-9', metadata: { snippet: true } },
|
||||
testDb
|
||||
);
|
||||
const [row] = await testDb.select().from(analyticsEvents);
|
||||
expect(row.userId).toBe('user-9');
|
||||
expect(JSON.parse(row.metadata!)).toEqual({ snippet: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
activePlayers,
|
||||
newVsReturning,
|
||||
netGrowth,
|
||||
d1Return,
|
||||
returnWithin,
|
||||
overallReturnRate,
|
||||
streakSurvival,
|
||||
timeToSecondPlayHistogram,
|
||||
guessHistogram,
|
||||
firstTryRate,
|
||||
averageGuesses,
|
||||
entropy,
|
||||
repeatIntervals,
|
||||
bookCoverage,
|
||||
retentionByGuessBucket,
|
||||
weekdayHourHeatmap,
|
||||
cohortTriangle,
|
||||
dailyCompletionsSeries
|
||||
} from '$lib/server/stats/metrics';
|
||||
import type { CompletionRow } from '$lib/types/stats';
|
||||
|
||||
function c(anonymousId: string, date: string, guessCount = 3, hour = 12): CompletionRow {
|
||||
return {
|
||||
anonymousId,
|
||||
date,
|
||||
guessCount,
|
||||
guesses: null,
|
||||
completedAt: new Date(`${date}T${String(hour).padStart(2, '0')}:00:00Z`)
|
||||
};
|
||||
}
|
||||
|
||||
// A: 3-day streak. B: plays day 1 and 3. C: one play. D: one play later.
|
||||
const fixture: CompletionRow[] = [
|
||||
c('A', '2026-09-01'),
|
||||
c('A', '2026-09-02'),
|
||||
c('A', '2026-09-03'),
|
||||
c('B', '2026-09-01'),
|
||||
c('B', '2026-09-03'),
|
||||
c('C', '2026-09-02', 1),
|
||||
c('D', '2026-09-05', 6)
|
||||
];
|
||||
|
||||
describe('activePlayers', () => {
|
||||
test('window of 1 is just that day', () => {
|
||||
expect(activePlayers(fixture, '2026-09-03', 1)).toBe(2); // A, B
|
||||
});
|
||||
test('window includes the trailing days', () => {
|
||||
expect(activePlayers(fixture, '2026-09-03', 3)).toBe(3); // A, B, C
|
||||
});
|
||||
test('empty window returns 0', () => {
|
||||
expect(activePlayers(fixture, '2026-08-01', 1)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('newVsReturning', () => {
|
||||
test('day one is all new', () => {
|
||||
expect(newVsReturning(fixture, '2026-09-01')).toEqual({
|
||||
newPlayers: 2,
|
||||
returning: 0,
|
||||
returningShare: 0
|
||||
});
|
||||
});
|
||||
test('later days are all returning', () => {
|
||||
expect(newVsReturning(fixture, '2026-09-03')).toEqual({
|
||||
newPlayers: 0,
|
||||
returning: 2,
|
||||
returningShare: 100
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('return rates', () => {
|
||||
test('d1Return pools the cohort of first-timers', () => {
|
||||
// A first on 09-01 and returned 09-02; B first on 09-01 and did not.
|
||||
expect(d1Return(fixture, '2026-09-01')).toBe(50);
|
||||
});
|
||||
test('returnWithin counts any return in the window', () => {
|
||||
expect(returnWithin(fixture, '2026-09-01', 2)).toBe(100);
|
||||
});
|
||||
test('small cohorts return null when minSize is set', () => {
|
||||
expect(returnWithin(fixture, '2026-09-05', 1, 3)).toBeNull();
|
||||
});
|
||||
test('overall return rate = players with 2+ plays', () => {
|
||||
expect(overallReturnRate(fixture)).toBe(50); // A & B of A,B,C,D
|
||||
});
|
||||
});
|
||||
|
||||
describe('netGrowth', () => {
|
||||
test('new + resurrected - churned over the window', () => {
|
||||
// Window 09-04..09-05: D is new; A,B,C played the preceding window and stopped.
|
||||
const g = netGrowth(fixture, '2026-09-05', 2, 30);
|
||||
expect(g.newPlayers).toBe(1);
|
||||
expect(g.churned).toBe(3);
|
||||
expect(g.netGrowth).toBe(-2);
|
||||
});
|
||||
test('a long gap counts as resurrected', () => {
|
||||
const data = [...fixture, c('E', '2026-06-01'), c('E', '2026-09-05')];
|
||||
const g = netGrowth(data, '2026-09-05', 2, 30);
|
||||
expect(g.resurrected).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('streaks', () => {
|
||||
test('survival probability per streak length', () => {
|
||||
const survival = streakSurvival(fixture);
|
||||
const byLength = new Map(survival.map((s) => [s.length, s.probability]));
|
||||
expect(byLength.get(1)).toBe(20); // only A continued from day 1
|
||||
expect(byLength.get(2)).toBe(100); // A continued to day 3
|
||||
expect(byLength.get(3)).toBe(0); // A stopped after day 3
|
||||
});
|
||||
});
|
||||
|
||||
describe('histograms', () => {
|
||||
test('time to second play buckets gaps', () => {
|
||||
const bins = timeToSecondPlayHistogram(fixture);
|
||||
expect(bins.find((b) => b.label === '1 day')?.value).toBe(1); // A
|
||||
expect(bins.find((b) => b.label === '2 days')?.value).toBe(1); // B
|
||||
});
|
||||
test('guess histogram collapses the tail', () => {
|
||||
const bins = guessHistogram(fixture, 5);
|
||||
const total = bins.reduce((a, b) => a + b.value, 0);
|
||||
expect(total).toBe(fixture.length);
|
||||
expect(bins[bins.length - 1].label).toBe('6+');
|
||||
});
|
||||
test('first try rate and average guesses', () => {
|
||||
expect(firstTryRate(fixture)).toBeCloseTo(14.3, 1);
|
||||
expect(typeof averageGuesses(fixture)).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('content metrics', () => {
|
||||
test('entropy of a uniform distribution is log2(n)', () => {
|
||||
expect(entropy([1, 1, 1, 1])).toBe(2);
|
||||
expect(entropy([])).toBe(0);
|
||||
});
|
||||
test('repeat intervals count gaps per book', () => {
|
||||
const bins = repeatIntervals([
|
||||
{ date: '2026-09-01', bookId: 'GEN' },
|
||||
{ date: '2026-09-03', bookId: 'GEN' },
|
||||
{ date: '2026-10-01', bookId: 'EXO' },
|
||||
{ date: '2026-10-05', bookId: 'EXO' }
|
||||
]);
|
||||
expect(bins.find((b) => b.label === '1–7d')?.value).toBe(2);
|
||||
});
|
||||
test('bookCoverage returns a stat for every book', () => {
|
||||
const stats = bookCoverage(
|
||||
[
|
||||
{ date: '2026-09-01', bookId: 'GEN' },
|
||||
{ date: '2026-09-02', bookId: 'GEN' }
|
||||
],
|
||||
fixture,
|
||||
{ start: '2026-09-01', end: '2026-09-03' }
|
||||
);
|
||||
expect(stats).toHaveLength(66);
|
||||
const gen = stats.find((s) => s.bookId === 'GEN')!;
|
||||
expect(gen.days).toBe(2);
|
||||
expect(gen.plays).toBe(4); // A,B on 09-01 + A,C on 09-02
|
||||
});
|
||||
test('retention by guess bucket is grouped', () => {
|
||||
const buckets = retentionByGuessBucket(fixture);
|
||||
expect(buckets).toHaveLength(5);
|
||||
});
|
||||
test('weekday/hour heatmap has a 7x24 grid', () => {
|
||||
const grid = weekdayHourHeatmap(fixture);
|
||||
expect(grid).toHaveLength(7);
|
||||
expect(grid[0]).toHaveLength(24);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cohortTriangle', () => {
|
||||
test('week 0 is the full cohort and later weeks are subsets', () => {
|
||||
const rows = cohortTriangle(fixture, {
|
||||
start: '2026-08-30', // Sunday
|
||||
weeks: 3,
|
||||
asOf: '2026-09-15',
|
||||
minSize: 1
|
||||
});
|
||||
expect(rows).toHaveLength(3);
|
||||
expect(rows[0].size).toBeGreaterThan(0);
|
||||
expect(rows[0].counts[0]).toBe(rows[0].size);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dailyCompletionsSeries', () => {
|
||||
test('fills gaps with zeroes and counts new players', () => {
|
||||
const series = dailyCompletionsSeries(fixture, '2026-09-01', '2026-09-03');
|
||||
expect(series).toHaveLength(3);
|
||||
expect(series[0]).toEqual({ date: '2026-09-01', completions: 2, newPlayers: 2 });
|
||||
expect(series[2]).toEqual({ date: '2026-09-03', completions: 2, newPlayers: 0 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { deriveSegments, countSegments } from '$lib/server/stats/segments';
|
||||
import { SETTINGS_DEFAULTS, type PlayerState, type StatsSettings } from '$lib/types/stats';
|
||||
import { addDays } from '$lib/server/stats/dates';
|
||||
|
||||
const AS_OF = '2026-09-15';
|
||||
|
||||
function settings(overrides: Partial<StatsSettings> = {}): StatsSettings {
|
||||
return { ...SETTINGS_DEFAULTS, ...overrides };
|
||||
}
|
||||
|
||||
function player(id: string, overrides: Partial<PlayerState>): PlayerState {
|
||||
return {
|
||||
id,
|
||||
firstDate: addDays(AS_OF, -100),
|
||||
lastDate: AS_OF,
|
||||
totalPlays: 10,
|
||||
plays7: 2,
|
||||
plays30: 10,
|
||||
streak: 2,
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
const players: PlayerState[] = [
|
||||
player('new', { firstDate: addDays(AS_OF, -1), lastDate: AS_OF, plays7: 1 }),
|
||||
player('core', { lastDate: AS_OF, plays7: 5 }),
|
||||
player('resurrected', { lastDate: AS_OF, plays7: 2, plays30: 2 }),
|
||||
player('returning', { lastDate: AS_OF, plays7: 2, plays30: 10 }),
|
||||
player('atRisk', { lastDate: addDays(AS_OF, -10) }),
|
||||
player('churned', { lastDate: addDays(AS_OF, -40) })
|
||||
];
|
||||
|
||||
describe('deriveSegments', () => {
|
||||
test('classifies each lifecycle segment', () => {
|
||||
const sets = deriveSegments(players, AS_OF, settings());
|
||||
expect(sets.new).toEqual(['new']);
|
||||
expect(sets.core).toEqual(['core']);
|
||||
expect(sets.resurrected).toEqual(['resurrected']);
|
||||
expect(sets.returning).toEqual(['returning']);
|
||||
expect(sets.atRisk).toEqual(['atRisk']);
|
||||
expect(sets.churned).toEqual(['churned']);
|
||||
expect(sets.active).toEqual(['new', 'core', 'resurrected', 'returning']);
|
||||
});
|
||||
|
||||
test('changing the core threshold re-segments without new data', () => {
|
||||
const strict = deriveSegments(players, AS_OF, settings({ core_plays_per_week: 6 }));
|
||||
expect(strict.core).toEqual([]);
|
||||
expect(strict.returning).toContain('core');
|
||||
|
||||
const lenient = deriveSegments(players, AS_OF, settings({ core_plays_per_week: 2 }));
|
||||
expect(lenient.core).toContain('core');
|
||||
expect(lenient.core).toContain('returning');
|
||||
});
|
||||
|
||||
test('counts include every player exactly once across mutually exclusive sets', () => {
|
||||
const counts = countSegments(deriveSegments(players, AS_OF, settings()));
|
||||
const activeTotal = counts.new + counts.core + counts.returning + counts.resurrected;
|
||||
expect(activeTotal).toBe(counts.active);
|
||||
expect(counts.total).toBe(players.length);
|
||||
});
|
||||
|
||||
test('at-risk and churned boundaries follow settings', () => {
|
||||
const sets = deriveSegments(players, AS_OF, settings({ at_risk_days: 5, churn_days: 20 }));
|
||||
expect(sets.atRisk).toContain('atRisk'); // inactive 10 >= 5, < 20
|
||||
expect(sets.churned).toContain('churned'); // inactive 40 >= 20
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { testDb } from '$lib/server/db/test';
|
||||
import { dashboardSettings } from '$lib/server/db/schema';
|
||||
import {
|
||||
getSettings,
|
||||
updateSettings,
|
||||
hashSettings,
|
||||
envSettings,
|
||||
settingsEnvVar
|
||||
} from '$lib/server/stats/settings';
|
||||
import { SETTINGS_DEFAULTS } from '$lib/types/stats';
|
||||
|
||||
async function clear() {
|
||||
await testDb.delete(dashboardSettings);
|
||||
}
|
||||
|
||||
describe('stats settings', () => {
|
||||
beforeEach(clear);
|
||||
afterEach(clear);
|
||||
|
||||
test('env var names map from keys', () => {
|
||||
expect(settingsEnvVar('d1_target')).toBe('STATS_D1_TARGET');
|
||||
});
|
||||
|
||||
test('empty table returns built-in defaults', async () => {
|
||||
const settings = await getSettings(testDb);
|
||||
expect(settings).toEqual({ ...SETTINGS_DEFAULTS });
|
||||
});
|
||||
|
||||
test('updateSettings upserts and is reflected on re-read', async () => {
|
||||
await updateSettings({ core_plays_per_week: 7 }, testDb);
|
||||
const settings = await getSettings(testDb);
|
||||
expect(settings.core_plays_per_week).toBe(7);
|
||||
// Updating again overwrites rather than duplicating.
|
||||
await updateSettings({ core_plays_per_week: 2 }, testDb);
|
||||
expect((await getSettings(testDb)).core_plays_per_week).toBe(2);
|
||||
});
|
||||
|
||||
test('config hash is stable and changes with settings', () => {
|
||||
const a = hashSettings({ ...SETTINGS_DEFAULTS });
|
||||
const b = hashSettings({ ...SETTINGS_DEFAULTS });
|
||||
const c = hashSettings({ ...SETTINGS_DEFAULTS, d1_target: 99 });
|
||||
expect(a).toBe(b);
|
||||
expect(a).not.toBe(c);
|
||||
expect(a).toHaveLength(12);
|
||||
});
|
||||
|
||||
test('envSettings falls back to defaults with no env overrides', () => {
|
||||
expect(envSettings()).toEqual({ ...SETTINGS_DEFAULTS });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { evaluateSignals, detectAnomaly, sortSignals } from '$lib/server/stats/signals';
|
||||
import { SETTINGS_DEFAULTS, type SnapshotPayload } from '$lib/types/stats';
|
||||
|
||||
function makePayload(overrides: {
|
||||
kpis?: Partial<SnapshotPayload['kpis']>;
|
||||
activity?: Partial<SnapshotPayload['activity']>;
|
||||
dailySeries?: SnapshotPayload['dailySeries'];
|
||||
} = {}): SnapshotPayload {
|
||||
return {
|
||||
asOfDate: '2026-09-14',
|
||||
activity: {
|
||||
dau: 100,
|
||||
wau: 300,
|
||||
mau: 800,
|
||||
core: 50,
|
||||
returningShare: 40,
|
||||
newPlayers: 20,
|
||||
resurrected: 5,
|
||||
churned: 18,
|
||||
netGrowth: 7,
|
||||
corePrevious: 45,
|
||||
previous: { dau: 95, wau: 290, mau: 790, core: 45 },
|
||||
...overrides.activity
|
||||
},
|
||||
dailySeries: overrides.dailySeries ?? [],
|
||||
kpis: {
|
||||
d1Return: 25,
|
||||
return7: 30,
|
||||
overallReturnRate: 22,
|
||||
completionRate: 80,
|
||||
shareRate: 30,
|
||||
firstTryRate: 25,
|
||||
avgGuesses: 4.1,
|
||||
...overrides.kpis
|
||||
},
|
||||
retention: {
|
||||
cohorts: [],
|
||||
curves: { d1: [], d7: [], d30: [] },
|
||||
timeToSecondPlay: [],
|
||||
segmentMix: []
|
||||
},
|
||||
growth: { lifecycleFunnel: [], signups: [] },
|
||||
totals: { completions: 0, players: 0, yesterdayCompletions: 0, yesterdayAvgGuesses: null },
|
||||
verseMix: {
|
||||
coverage: [],
|
||||
testamentMix: [],
|
||||
entropy: [],
|
||||
repeatIntervals: [],
|
||||
firstTryByBook: [],
|
||||
retentionByBook: [],
|
||||
retentionByGuessBucket: [],
|
||||
achievability: []
|
||||
},
|
||||
habit: { streakSurvival: [], streakLengths: [], calendar: [], weekdayHour: [] },
|
||||
signals: [],
|
||||
meta: { hasStartedData: true, hasShareData: true, firstEventDate: null }
|
||||
};
|
||||
}
|
||||
|
||||
const settings = { ...SETTINGS_DEFAULTS };
|
||||
|
||||
describe('evaluateSignals', () => {
|
||||
test('all-good payload yields no warn/bad signals', () => {
|
||||
const signals = evaluateSignals(makePayload(), settings);
|
||||
expect(signals.some((s) => s.severity === 'bad' || s.severity === 'warn')).toBe(false);
|
||||
});
|
||||
|
||||
test('low D1 return is bad and recommends an action', () => {
|
||||
const signals = evaluateSignals(makePayload({ kpis: { d1Return: 9 } }), settings);
|
||||
const d1 = signals.find((s) => s.id === 'd1_return')!;
|
||||
expect(d1.severity).toBe('bad');
|
||||
expect(d1.action).toBeTruthy();
|
||||
});
|
||||
|
||||
test('missed-but-not-red targets are warnings', () => {
|
||||
const signals = evaluateSignals(makePayload({ kpis: { d1Return: 16 } }), settings);
|
||||
expect(signals.find((s) => s.id === 'd1_return')!.severity).toBe('warn');
|
||||
});
|
||||
|
||||
test('first-try rate outside the band is flagged', () => {
|
||||
const tooEasy = evaluateSignals(makePayload({ kpis: { firstTryRate: 60 } }), settings);
|
||||
expect(tooEasy.find((s) => s.id === 'first_try')!.severity).toBe('bad');
|
||||
const tooHard = evaluateSignals(makePayload({ kpis: { firstTryRate: 5 } }), settings);
|
||||
expect(tooHard.find((s) => s.id === 'first_try')!.severity).toBe('bad');
|
||||
});
|
||||
|
||||
test('missing metrics surface as info, not failure', () => {
|
||||
const signals = evaluateSignals(
|
||||
makePayload({ kpis: { completionRate: null, shareRate: null } }),
|
||||
settings
|
||||
);
|
||||
expect(signals.find((s) => s.id === 'completion')!.severity).toBe('info');
|
||||
expect(signals.find((s) => s.id === 'share_rate')!.severity).toBe('info');
|
||||
});
|
||||
|
||||
test('declining core players is a warning', () => {
|
||||
const signals = evaluateSignals(
|
||||
makePayload({ activity: { core: 30, corePrevious: 50 } }),
|
||||
settings
|
||||
);
|
||||
expect(signals.find((s) => s.id === 'core_players')!.severity).toBe('warn');
|
||||
});
|
||||
|
||||
test('sortSignals puts bad first', () => {
|
||||
const signals = sortSignals(evaluateSignals(makePayload({ kpis: { d1Return: 5 } }), settings));
|
||||
expect(signals[0].severity).toBe('bad');
|
||||
});
|
||||
});
|
||||
|
||||
describe('detectAnomaly', () => {
|
||||
function series(values: number[], startWeekdayOffset = 0) {
|
||||
// Build consecutive days starting 2026-08-01 so weekdays repeat every 7.
|
||||
const out = values.map((value, i) => {
|
||||
const d = new Date('2026-08-01T00:00:00Z');
|
||||
d.setUTCDate(d.getUTCDate() + i);
|
||||
return { date: d.toISOString().slice(0, 10), value };
|
||||
});
|
||||
void startWeekdayOffset;
|
||||
return out;
|
||||
}
|
||||
|
||||
test('flags a large weekday-aware spike', () => {
|
||||
// 5 weeks of varying same-weekday values, then a spike.
|
||||
const base: number[] = [];
|
||||
for (let i = 0; i < 35; i++) base.push(100 + ((i * 13) % 17));
|
||||
const spike = [...base, 500];
|
||||
const anomaly = detectAnomaly(series(spike), 2);
|
||||
expect(anomaly).not.toBeNull();
|
||||
expect(anomaly!.value).toBe(500);
|
||||
});
|
||||
|
||||
test('returns null for a flat series', () => {
|
||||
expect(detectAnomaly(series(Array(35).fill(100)), 2)).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null without enough history', () => {
|
||||
expect(detectAnomaly(series([1, 2, 3]), 2)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { testDb } from '$lib/server/db/test';
|
||||
import {
|
||||
analyticsEvents,
|
||||
dailyCompletions,
|
||||
dailyVerses,
|
||||
statsSnapshot
|
||||
} from '$lib/server/db/schema';
|
||||
import {
|
||||
buildSnapshot,
|
||||
computePayload,
|
||||
computePlayerState,
|
||||
ensureSnapshot,
|
||||
PAYLOAD_SCHEMA_VERSION,
|
||||
type LoadedData
|
||||
} from '$lib/server/stats/snapshot';
|
||||
import { SETTINGS_DEFAULTS, type CompletionRow } from '$lib/types/stats';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const settings = { ...SETTINGS_DEFAULTS };
|
||||
|
||||
function c(anonymousId: string, date: string): CompletionRow {
|
||||
return { anonymousId, date, guessCount: 3, guesses: null, completedAt: new Date(date) };
|
||||
}
|
||||
|
||||
describe('computePlayerState', () => {
|
||||
test('builds streak and rolling play counts as of the date', () => {
|
||||
const completions: CompletionRow[] = [
|
||||
c('A', '2026-09-08'),
|
||||
c('A', '2026-09-09'),
|
||||
c('A', '2026-09-10'),
|
||||
c('A', '2026-08-01'),
|
||||
c('B', '2026-09-10')
|
||||
];
|
||||
const state = computePlayerState(completions, '2026-09-10', 180);
|
||||
const a = state.find((p) => p.id === 'A')!;
|
||||
expect(a.firstDate).toBe('2026-08-01');
|
||||
expect(a.lastDate).toBe('2026-09-10');
|
||||
expect(a.totalPlays).toBe(4);
|
||||
expect(a.plays7).toBe(3);
|
||||
expect(a.streak).toBe(3);
|
||||
});
|
||||
|
||||
test('drops players outside the active window', () => {
|
||||
const state = computePlayerState([c('old', '2026-01-01')], '2026-09-10', 30);
|
||||
expect(state).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computePayload point-in-time cutoff', () => {
|
||||
const data: LoadedData = {
|
||||
completions: [
|
||||
c('A', '2026-09-10'),
|
||||
c('A', '2026-09-11'), // future relative to asOfDate
|
||||
c('B', '2026-09-11') // future
|
||||
],
|
||||
verses: [{ date: '2026-09-10', bookId: 'GEN' }],
|
||||
startedByDate: new Map([['2026-09-10', 2]]),
|
||||
shareEvents: 0,
|
||||
firstEventDate: '2026-09-10',
|
||||
signupDates: []
|
||||
};
|
||||
|
||||
test('activity only reflects data up to the as-of date', () => {
|
||||
const payload = computePayload('2026-09-10', settings, data);
|
||||
expect(payload.activity.dau).toBe(1); // only A, not B (future)
|
||||
});
|
||||
|
||||
test('completion rate uses verse_served events', () => {
|
||||
const payload = computePayload('2026-09-10', settings, data);
|
||||
expect(payload.meta.hasStartedData).toBe(true);
|
||||
expect(payload.kpis.completionRate).not.toBeNull();
|
||||
});
|
||||
|
||||
test('payload has all lab sections', () => {
|
||||
const payload = computePayload('2026-09-10', settings, data);
|
||||
expect(payload.retention.cohorts.length).toBeGreaterThan(0);
|
||||
expect(payload.verseMix.coverage).toHaveLength(66);
|
||||
expect(payload.habit.calendar.length).toBe(120);
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapshot persistence', () => {
|
||||
beforeEach(async () => {
|
||||
await testDb.delete(statsSnapshot);
|
||||
await testDb.delete(analyticsEvents);
|
||||
await testDb.delete(dailyCompletions);
|
||||
await testDb.delete(dailyVerses);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await testDb.delete(statsSnapshot);
|
||||
await testDb.delete(analyticsEvents);
|
||||
await testDb.delete(dailyCompletions);
|
||||
await testDb.delete(dailyVerses);
|
||||
});
|
||||
|
||||
async function seed() {
|
||||
await testDb.insert(dailyCompletions).values([
|
||||
{ id: crypto.randomUUID(), anonymousId: 'A', date: '2026-01-10', guessCount: 2, completedAt: new Date('2026-01-10T12:00:00Z') },
|
||||
{ id: crypto.randomUUID(), anonymousId: 'A', date: '2026-01-11', guessCount: 3, completedAt: new Date('2026-01-11T12:00:00Z') },
|
||||
{ id: crypto.randomUUID(), anonymousId: 'B', date: '2026-01-11', guessCount: 1, completedAt: new Date('2026-01-11T13:00:00Z') }
|
||||
]);
|
||||
await testDb.insert(dailyVerses).values([
|
||||
{ id: crypto.randomUUID(), date: '2026-01-10', bookId: 'GEN', verseText: 'x', reference: 'Genesis 1:1' },
|
||||
{ id: crypto.randomUUID(), date: '2026-01-11', bookId: 'EXO', verseText: 'y', reference: 'Exodus 1:1' }
|
||||
]);
|
||||
}
|
||||
|
||||
test('building the same date twice is idempotent', async () => {
|
||||
await seed();
|
||||
const first = await buildSnapshot('2026-01-10', { kind: 'lazy' }, testDb);
|
||||
await buildSnapshot('2026-01-10', { kind: 'lazy' }, testDb);
|
||||
const rows = await testDb.select().from(statsSnapshot);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(first.snapshot.payload.asOfDate).toBe('2026-01-10');
|
||||
expect(first.snapshot.payload.activity.dau).toBe(1);
|
||||
});
|
||||
|
||||
test('ensureSnapshot builds when missing and reads when present', async () => {
|
||||
await seed();
|
||||
const a = await ensureSnapshot('2026-01-10', {}, testDb);
|
||||
const b = await ensureSnapshot('2026-01-10', {}, testDb);
|
||||
expect(a.generatedAt.getTime()).toBe(b.generatedAt.getTime());
|
||||
expect(a.buildKind).toBe('lazy');
|
||||
});
|
||||
|
||||
test('rebuilds snapshots written with an outdated payload shape', async () => {
|
||||
await seed();
|
||||
await testDb.insert(statsSnapshot).values({
|
||||
date: '2026-01-10',
|
||||
generatedAt: new Date(),
|
||||
buildKind: 'lazy',
|
||||
configVersion: 'outdated',
|
||||
payload: '{}',
|
||||
playerState: '[]',
|
||||
config: '{}'
|
||||
});
|
||||
const snap = await ensureSnapshot('2026-01-10', {}, testDb);
|
||||
expect(snap.payload.meta?.schemaVersion).toBe(PAYLOAD_SCHEMA_VERSION);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user