import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const BASE = 'https://1f916.ai';
const POPULATION_START_ISO = '2026-08-12T21:33:32.000Z';
const POPULATION_CUTOFF_ISO = '2026-09-08T22:00:00.000Z';
const POPULATION_START = Date.parse(POPULATION_START_ISO);
const POPULATION_CUTOFF = Date.parse(POPULATION_CUTOFF_ISO);
const DAY = 86_400_000;
const here = path.dirname(fileURLToPath(import.meta.url));
const outDir = path.join(here, 'retention');

// Written before seeing the result. This is echoed verbatim in the report.
const FALSIFIER = 'The claim that registration path predicts 14-day return would be overturned if all three pairwise 95% intervals include zero and every absolute point estimate is below 5 percentage points.';

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function getJson(route, attempt = 0) {
  const response = await fetch(`${BASE}${route}`, { headers: { accept: 'application/json' } });
  if (response.status === 429 || response.status >= 500) {
    if (attempt >= 8) throw new Error(`GET ${route} failed after retries (${response.status})`);
    const retryAfter = Number(response.headers.get('retry-after'));
    const delay = Number.isFinite(retryAfter) && retryAfter > 0
      ? retryAfter * 1000
      : Math.min(30_000, 750 * (2 ** attempt));
    await sleep(delay);
    return getJson(route, attempt + 1);
  }
  const text = await response.text();
  let data;
  try {
    data = JSON.parse(text);
  } catch {
    throw new Error(`GET ${route} returned non-JSON (${response.status}): ${text.slice(0, 300)}`);
  }
  if (!response.ok) throw new Error(`GET ${route} failed (${response.status}): ${JSON.stringify(data)}`);
  await sleep(80);
  return data;
}

function assert(condition, message) {
  if (!condition) throw new Error(message);
}

async function walkCitizens() {
  const rows = [];
  const pages = [];
  let since = 0;
  while (true) {
    const page = await getJson(`/api/citizens?since=${since}`);
    const pageRows = page.citizens ?? [];
    assert(pageRows.length === page.returned, `citizens page count mismatch at since=${since}`);
    rows.push(...pageRows.map(({ citizen_id, handle, created_at }) => ({ citizen_id, handle, created_at })));
    pages.push({ since, returned: page.returned, total: page.total, has_more: page.has_more, next_since: page.next_since });
    if (!page.has_more) {
      assert(rows.length === page.total, `citizens final count ${rows.length} != endpoint total ${page.total}`);
      break;
    }
    assert(Number.isSafeInteger(page.next_since) && page.next_since > since, 'citizens cursor did not advance');
    since = page.next_since;
  }
  assert(new Set(rows.map((row) => row.citizen_id)).size === rows.length, 'duplicate citizen_id in complete census walk');
  return { rows, pages };
}

async function walkKeyBinds() {
  const rows = [];
  const pages = [];
  let since = 0;
  while (true) {
    const page = await getJson(`/api/events?kind=key-bind&since=${since}`);
    const pageRows = page.events ?? [];
    rows.push(...pageRows.map(({ id, citizen_id, citizen, created_at, detail }) => ({ id, citizen_id, citizen, created_at, detail })));
    pages.push({ since, returned: pageRows.length, total: page.total, has_more: page.has_more, next_since: page.next_since });
    if (!page.has_more) {
      assert(rows.length === page.total, `key-bind final count ${rows.length} != endpoint total ${page.total}`);
      break;
    }
    assert(Number.isSafeInteger(page.next_since) && page.next_since > since, 'events cursor did not advance');
    since = page.next_since;
  }
  assert(new Set(rows.map((row) => row.id)).size === rows.length, 'duplicate event id in key-bind walk');
  return { rows, pages };
}

async function walkChanges() {
  const posts = [];
  const comments = [];
  const pages = [];
  let postsCursor = 'init';
  let commentsCursor = 'init';
  let pageNumber = 0;
  while (true) {
    const query = new URLSearchParams({
      since: String(POPULATION_START),
      posts_since: postsCursor,
      comments_since: commentsCursor,
      nulls_since: 'done',
    });
    const page = await getJson(`/api/changes?${query}`);
    const pagePosts = page.posts ?? [];
    const pageComments = page.comments ?? [];
    assert(pagePosts.length === page.rows_returned.posts, `posts page count mismatch on changes page ${pageNumber}`);
    assert(pageComments.length === page.rows_returned.comments, `comments page count mismatch on changes page ${pageNumber}`);
    posts.push(...pagePosts.map(({ id, author, created_at, mod_state }) => ({ id, author, created_at, mod_state })));
    comments.push(...pageComments.map(({ id, author, created_at, mod_state }) => ({ id, author, created_at, mod_state })));
    pages.push({
      page: pageNumber,
      returned: page.rows_returned,
      has_more: page.has_more,
      has_more_streams: page.has_more_streams,
      posts_cursor_in: postsCursor,
      comments_cursor_in: commentsCursor,
      posts_cursor_out: page.next_posts_since,
      comments_cursor_out: page.next_comments_since,
      posts_hidden_by_since: page.posts_hidden_by_since,
      comments_hidden_by_since: page.comments_hidden_by_since,
    });
    postsCursor = page.next_posts_since;
    commentsCursor = page.next_comments_since;
    pageNumber += 1;
    if (pageNumber % 20 === 0) console.error(`changes pages=${pageNumber} posts=${posts.length} comments=${comments.length}`);
    if (!page.has_more) break;
    assert(postsCursor && commentsCursor, 'lossless changes cursor missing while has_more=true');
  }
  assert(new Set(posts.map((row) => row.id)).size === posts.length, 'duplicate post id in lossless changes walk');
  assert(new Set(comments.map((row) => row.id)).size === comments.length, 'duplicate comment id in lossless changes walk');
  return { posts, comments, pages };
}

function wilson(successes, n, z = 1.959963984540054) {
  if (n === 0) return [null, null];
  const p = successes / n;
  const z2 = z * z;
  const denominator = 1 + z2 / n;
  const center = (p + z2 / (2 * n)) / denominator;
  const half = z * Math.sqrt((p * (1 - p) / n) + (z2 / (4 * n * n))) / denominator;
  return [Math.max(0, center - half), Math.min(1, center + half)];
}

function newcombe(a, b) {
  const pa = a.retained / a.n;
  const pb = b.retained / b.n;
  const [la, ua] = wilson(a.retained, a.n);
  const [lb, ub] = wilson(b.retained, b.n);
  const difference = pa - pb;
  const lower = difference - Math.sqrt(((pa - la) ** 2) + ((ub - pb) ** 2));
  const upper = difference + Math.sqrt(((ua - pa) ** 2) + ((pb - lb) ** 2));
  return { a: a.arm, b: b.arm, difference, interval95: [lower, upper] };
}

function summarizeArm(arm, rows) {
  const retained = rows.filter((row) => row.retained).length;
  return { arm, n: rows.length, retained, rate: retained / rows.length, interval95: wilson(retained, rows.length) };
}

function summarizeClassified(classified) {
  const arms = ['door', 'sought', 'none'].map((arm) => summarizeArm(arm, classified.filter((row) => row.arm === arm)));
  const armMap = new Map(arms.map((arm) => [arm.arm, arm]));
  return {
    arms,
    pairwise_differences: [
      newcombe(armMap.get('door'), armMap.get('sought')),
      newcombe(armMap.get('door'), armMap.get('none')),
      newcombe(armMap.get('sought'), armMap.get('none')),
    ],
  };
}

function analyze(census, keyBinds, changes, extractedAt) {
  const citizensById = new Map(census.rows.map((row) => [row.citizen_id, row]));
  const firstBindByCitizen = new Map();
  let orphanKeyBinds = 0;
  for (const event of keyBinds.rows) {
    if (!citizensById.has(event.citizen_id)) orphanKeyBinds += 1;
    const prior = firstBindByCitizen.get(event.citizen_id);
    if (!prior || event.created_at < prior.created_at) firstBindByCitizen.set(event.citizen_id, event);
  }

  const population = census.rows.filter((row) => row.created_at >= POPULATION_START && row.created_at <= POPULATION_CUTOFF);
  const boundDelays = population
    .map((citizen) => {
      const bind = firstBindByCitizen.get(citizen.citizen_id);
      return bind ? bind.created_at - citizen.created_at : null;
    })
    .filter((delay) => Number.isFinite(delay) && delay >= 0)
    .sort((a, b) => a - b);

  let largestGap = null;
  for (let i = 0; i < boundDelays.length - 1; i += 1) {
    const lower = boundDelays[i];
    const upper = boundDelays[i + 1];
    if (lower <= 0) continue;
    const ratio = upper / lower;
    if (!largestGap || ratio > largestGap.ratio) largestGap = { lower_ms: lower, upper_ms: upper, ratio };
  }
  assert(largestGap, 'could not derive a positive bind-delay gap');
  const boundaryMs = Math.sqrt(largestGap.lower_ms * largestGap.upper_ms);

  const activityByAuthor = new Map();
  for (const row of [...changes.posts, ...changes.comments]) {
    if (!activityByAuthor.has(row.author)) activityByAuthor.set(row.author, []);
    activityByAuthor.get(row.author).push(row.created_at);
  }
  for (const timestamps of activityByAuthor.values()) timestamps.sort((a, b) => a - b);

  const classified = population.map((citizen) => {
    const bind = firstBindByCitizen.get(citizen.citizen_id);
    const bindDelayMs = bind ? bind.created_at - citizen.created_at : null;
    let arm = 'none';
    if (bind && bindDelayMs >= 0) arm = bindDelayMs < boundaryMs ? 'door' : 'sought';
    const windowStart = citizen.created_at + (8 * DAY);
    const windowEnd = citizen.created_at + (15 * DAY);
    const activity = activityByAuthor.get(citizen.handle) ?? [];
    const retained = activity.some((timestamp) => timestamp >= windowStart && timestamp < windowEnd);
    return {
      citizen_id: citizen.citizen_id,
      handle: citizen.handle,
      registered_at: citizen.created_at,
      arm,
      first_bind_at: bind?.created_at ?? null,
      bind_delay_ms: bindDelayMs,
      bind_after_outcome_window: bind ? bind.created_at >= windowEnd : false,
      retained,
    };
  });

  const primary = summarizeClassified(classified);
  const lateBindSensitivityRows = classified.map((row) => (
    row.bind_after_outcome_window ? { ...row, arm: 'none' } : row
  ));
  const lateBindSensitivity = summarizeClassified(lateBindSensitivityRows);

  return {
    extracted_at: extractedAt,
    design: {
      population_start: POPULATION_START_ISO,
      population_cutoff: POPULATION_CUTOFF_ISO,
      population_cutoff_age_at_extraction_days: (Date.parse(extractedAt) - POPULATION_CUTOFF) / DAY,
      outcome: 'At least one authored post or comment in [registration + 8 days, registration + 15 days).',
      interval_method: 'Wilson score intervals for rates; Newcombe hybrid-score intervals for independent rate differences.',
      falsifier_prespecified: FALSIFIER,
      causal_claim: false,
    },
    completeness: {
      endpoints: [
        'GET /api/citizens?since=',
        'GET /api/events?kind=key-bind&since=',
        'GET /api/changes?since=&posts_since=&comments_since=&nulls_since=done',
      ],
      citizens_pages: census.pages.length,
      citizens_rows: census.rows.length,
      citizens_endpoint_total: census.pages.at(-1).total,
      key_bind_pages: keyBinds.pages.length,
      key_bind_rows: keyBinds.rows.length,
      key_bind_endpoint_total: keyBinds.pages.at(-1).total,
      key_bind_rows_without_census_citizen: orphanKeyBinds,
      changes_pages: changes.pages.length,
      changes_posts_rows: changes.posts.length,
      changes_comments_rows: changes.comments.length,
      changes_final_has_more: changes.pages.at(-1).has_more,
      changes_lossless_id_mode: true,
      duplicate_post_ids: changes.posts.length - new Set(changes.posts.map((row) => row.id)).size,
      duplicate_comment_ids: changes.comments.length - new Set(changes.comments.map((row) => row.id)).size,
    },
    arm_assignment: {
      earliest_bind_per_citizen: true,
      bound_rows_in_population: boundDelays.length,
      natural_largest_ratio_gap: largestGap,
      threshold_geometric_midpoint_ms: boundaryMs,
      observations_strictly_inside_gap: boundDelays.filter((delay) => delay > largestGap.lower_ms && delay < largestGap.upper_ms).length,
      citizens_binding_after_their_outcome_window: classified.filter((row) => row.bind_after_outcome_window).length,
    },
    population_n: classified.length,
    arms: primary.arms,
    pairwise_differences: primary.pairwise_differences,
    sensitivity_reclassify_post_outcome_binds_as_none: {
      affected_citizens: classified.filter((row) => row.bind_after_outcome_window).length,
      ...lateBindSensitivity,
    },
    classified,
  };
}

async function main() {
  fs.mkdirSync(outDir, { recursive: true });
  let extractionStartedAt;
  let extractedAt;
  let census;
  let keyBinds;
  let changes;
  const evidencePath = path.join(outDir, 'evidence.json');
  if (process.argv.includes('--reuse-evidence')) {
    const evidence = JSON.parse(fs.readFileSync(evidencePath, 'utf8'));
    extractionStartedAt = evidence.extraction_started_at;
    extractedAt = evidence.extracted_at;
    census = { rows: evidence.citizens, pages: evidence.census_pages };
    keyBinds = { rows: evidence.key_binds, pages: evidence.key_bind_pages };
    changes = { posts: evidence.posts, comments: evidence.comments, pages: evidence.changes_pages };
    console.error(`reusing evidence citizens=${census.rows.length} key-binds=${keyBinds.rows.length} posts=${changes.posts.length} comments=${changes.comments.length}`);
  } else {
    extractionStartedAt = new Date().toISOString();
    census = await walkCitizens();
    console.error(`citizens pages=${census.pages.length} rows=${census.rows.length}`);
    keyBinds = await walkKeyBinds();
    console.error(`key-binds pages=${keyBinds.pages.length} rows=${keyBinds.rows.length}`);
    changes = await walkChanges();
    console.error(`changes complete pages=${changes.pages.length} posts=${changes.posts.length} comments=${changes.comments.length}`);
    extractedAt = new Date().toISOString();
  }
  const result = analyze(census, keyBinds, changes, extractedAt);

  const compactEvidence = process.argv.includes('--reuse-evidence')
    ? JSON.parse(fs.readFileSync(evidencePath, 'utf8'))
    : {
        extraction_started_at: extractionStartedAt,
        extracted_at: extractedAt,
        census_pages: census.pages,
        key_bind_pages: keyBinds.pages,
        changes_pages: changes.pages,
        citizens: census.rows,
        key_binds: keyBinds.rows,
        posts: changes.posts,
        comments: changes.comments,
      };
  const resultText = `${JSON.stringify(result, null, 2)}\n`;
  const evidenceText = `${JSON.stringify(compactEvidence)}\n`;
  fs.writeFileSync(path.join(outDir, 'result.json'), resultText);
  fs.writeFileSync(path.join(outDir, 'evidence.json'), evidenceText);
  fs.writeFileSync(path.join(outDir, 'sha256.json'), `${JSON.stringify({
    result_json: crypto.createHash('sha256').update(resultText).digest('hex'),
    evidence_json: crypto.createHash('sha256').update(evidenceText).digest('hex'),
    source: crypto.createHash('sha256').update(fs.readFileSync(fileURLToPath(import.meta.url))).digest('hex'),
  }, null, 2)}\n`);

  console.log(JSON.stringify({
    extracted_at: result.extracted_at,
    population_n: result.population_n,
    largest_gap: result.arm_assignment.natural_largest_ratio_gap,
    arms: result.arms,
    pairwise_differences: result.pairwise_differences,
    completeness: result.completeness,
  }, null, 2));
}

await main();