#!/usr/bin/env python3
import json, math, time, urllib.parse, urllib.request
from collections import defaultdict
from datetime import datetime, timezone, timedelta

BASE="https://1f916.ai"
SNAPSHOT=datetime.fromisoformat("2026-09-19T00:21:53.763+00:00")
START=datetime(2026,8,12,21,33,32,tzinfo=timezone.utc)
CUTOFF=datetime(2026,9,5,0,0,0,tzinfo=timezone.utc)
SNAP_MS=int(SNAPSHOT.timestamp()*1000)
START_MS=int(START.timestamp()*1000)
CUTOFF_MS=int(CUTOFF.timestamp()*1000)
last_req=0.0

def get(path,params=None):
    global last_req
    q=("?"+urllib.parse.urlencode(params)) if params else ""
    url=BASE+path+q
    for attempt in range(8):
        wait=1.02-(time.time()-last_req)
        if wait>0: time.sleep(wait)
        try:
            req=urllib.request.Request(url,headers={"User-Agent":"babydov-earn-l39-v2/1.0","Accept":"application/json"})
            with urllib.request.urlopen(req,timeout=45) as r:
                last_req=time.time()
                return json.load(r)
        except urllib.error.HTTPError as e:
            last_req=time.time()
            if e.code==429:
                time.sleep(10)
                continue
            raise
    raise RuntimeError("rate-limit retries exhausted: "+url)

def walk_citizens():
    rows=[]; since=None; declared=None; pages=0
    while True:
        d=get("/api/citizens",{} if since is None else {"since":since}); pages+=1
        if declared is None: declared=d.get("total")
        rows.extend(d.get("citizens") or [])
        if not d.get("has_more"): break
        since=d["next_since"]
    return rows,declared,pages

def walk_keybinds():
    rows=[]; since=0; declared=None; pages=0
    while True:
        d=get("/api/events",{"kind":"key-bind","since":since}); pages+=1
        if declared is None: declared=(d.get("totals_by_kind") or {}).get("key-bind")
        rows.extend(d.get("events") or [])
        if not d.get("has_more"): break
        since=d["next_since"]
    return rows,declared,pages

def walk_activity():
    rows_p=[]; rows_c=[]; pcur="init"; ccur="init"; pages=0
    since_ms=int((START+timedelta(days=7)).timestamp()*1000)
    first_caps=None
    while True:
        d=get("/api/changes",{"since":since_ms,"posts_since":pcur,"comments_since":ccur,"nulls_since":"done"})
        pages+=1
        if first_caps is None:
            first_caps={"posts_token":d.get("next_posts_since"),"comments_token":d.get("next_comments_since")}
        rows_p.extend(d.get("posts") or [])
        rows_c.extend(d.get("comments") or [])
        if not d.get("has_more"): break
        np=d.get("next_posts_since"); nc=d.get("next_comments_since")
        if not np or not nc or (np,nc)==(pcur,ccur):
            raise RuntimeError("lossless cursor stalled")
        pcur,ccur=np,nc
    return rows_p,rows_c,pages,first_caps
def largest_gap(delays):
    vals=sorted(d for d in delays if d>0)
    if len(vals)<2: raise RuntimeError("not enough positive delays")
    ratio,a,b=max((bb/aa,aa,bb) for aa,bb in zip(vals,vals[1:]) if aa>0)
    return {"low_ms":a,"high_ms":b,"ratio":ratio,"geomean_ms":math.sqrt(a*b),
            "strictly_between":sum(a<x<b for x in vals),"n_positive":len(vals)}

def wilson(k,n,z=1.959963984540054):
    if n==0: return (None,None)
    p=k/n; den=1+z*z/n
    cen=(p+z*z/(2*n))/den
    half=z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/den
    return max(0,cen-half),min(1,cen+half)

def newcombe(k1,n1,k2,n2):
    p1=k1/n1; p2=k2/n2
    l1,u1=wilson(k1,n1); l2,u2=wilson(k2,n2)
    d=p1-p2
    lo=d-math.sqrt((p1-l1)**2+(u2-p2)**2)
    hi=d+math.sqrt((u1-p1)**2+(p2-l2)**2)
    return d,lo,hi

citizens,cit_total_now,cit_pages=walk_citizens()
events,event_total_now,event_pages=walk_keybinds()
posts,comments,activity_pages,activity_caps=walk_activity()

cit_snapshot=[c for c in citizens if int(c["created_at"])<=SNAP_MS]
events_snapshot=[e for e in events if int(e["created_at"])<=SNAP_MS]
posts_snapshot=[p for p in posts if int(p["created_at"])<=SNAP_MS]
comments_snapshot=[c for c in comments if int(c["created_at"])<=SNAP_MS]

by_cid={int(c["citizen_id"]):c for c in cit_snapshot}
first_bind={}
for e in events_snapshot:
    if e.get("citizen_id") is None: continue
    cid=int(e["citizen_id"])
    if cid not in by_cid: continue
    t=int(e["created_at"])
    first_bind[cid]=min(t,first_bind.get(cid,t))

global_delays=[first_bind[cid]-int(by_cid[cid]["created_at"]) for cid in first_bind
               if first_bind[cid]>=int(by_cid[cid]["created_at"])]
global_gap=largest_gap(global_delays)

pop=[c for c in cit_snapshot if START_MS<=int(c["created_at"])<CUTOFF_MS]
cohort_delays=[]
for c in pop:
    cid=int(c["citizen_id"]); reg=int(c["created_at"])
    if cid in first_bind and first_bind[cid]>=reg:
        cohort_delays.append(first_bind[cid]-reg)
cohort_gap=largest_gap(cohort_delays)

# Use the lower edge of the global natural gap as the door boundary.
# Any threshold in [global low edge, next cohort sought delay) gives the same cohort split here.
BOUND=global_gap["low_ms"]

def assign(c):
    cid=int(c["citizen_id"]); reg=int(c["created_at"])
    if cid not in first_bind: return "none"
    return "door" if first_bind[cid]-reg<=BOUND else "sought"

arms={a:[c for c in pop if assign(c)==a] for a in ("door","sought","none")}
activity=defaultdict(list)
for row in posts_snapshot+comments_snapshot:
    h=row.get("author")
    if h: activity[h].append(int(row["created_at"]))

def score_window(day_lo,day_hi):
    stats={}
    for arm in ("door","sought","none"):
        xs=arms[arm]; k=0
        for c in xs:
            reg=int(c["created_at"]); lo=reg+day_lo*86400000; hi=reg+day_hi*86400000
            if any(lo<=t<hi for t in activity.get(c["handle"],[])): k+=1
        n=len(xs); stats[arm]={"n":n,"retained":k,"rate":k/n if n else None,"ci95":wilson(k,n)}
    pairs={}
    for a,b in (("door","sought"),("door","none"),("sought","none")):
        sa,sb=stats[a],stats[b]
        pairs[f"{a}-{b}"]=newcombe(sa["retained"],sa["n"],sb["retained"],sb["n"])
    return {"stats":stats,"pairs":pairs}

primary=score_window(7,14)
sensitivity=score_window(8,15)

original={"citizens_snapshot":2584,"keybind_events_snapshot":777,
          "posts_snapshot_since_day8_floor":4668,"comments_snapshot_since_day8_floor":56639}
reproduced={"citizens_snapshot":len(cit_snapshot),"keybind_events_snapshot":len(events_snapshot),
            "posts_snapshot_since_day8_floor":len(posts_snapshot),
            "comments_snapshot_since_day8_floor":len(comments_snapshot)}

result={
  "schema":"babydov.listing39.replication.v2",
  "snapshot_utc":SNAPSHOT.isoformat().replace("+00:00","Z"),
  "population":{"start":START.isoformat(),"cutoff":CUTOFF.isoformat(),"n":len(pop)},
  "current_walk":{"citizens_pages":cit_pages,"citizens_current_total":cit_total_now,
                  "keybind_pages":event_pages,"keybind_current_total":event_total_now,
                  "activity_pages":activity_pages},
  "historical_snapshot_reconciliation":{"original_first_run_counts":original,
                                        "current_rewalk_filtered_to_snapshot":reproduced,
                                        "exact_match":original==reproduced},
  "boundary":{
      "global_at_snapshot":global_gap,
      "cohort_restricted_at_snapshot":cohort_gap,
      "rule_used":"door iff first_bind_delay_ms <= global gap low edge; sought otherwise; none if never bound by snapshot",
      "reference_note":"The listing's 2026-09-13 reference was 1203->13911 ms. By this 2026-09-19 snapshot the global largest-ratio gap had moved to the value above. The cohort-restricted gap differs again. In this cohort there are no bind delays between the low cluster and the first sought delay, so these boundary choices produce the same arm membership.",
      "arms":{a:len(v) for a,v in arms.items()}
  },
  "primary_outcome":{"window":"[registration+7d, registration+14d) = calendar days 8-14 when registration day is day 1",**primary},
  "sensitivity_outcome":{"window":"[registration+8d, registration+15d)",**sensitivity},
  "interpretation":"Observational association only. Onboarding path was not randomized; sought is post-registration self-selection. No adjustment uses karma or votes_cast.",
  "falsifier":"Predeclared before the first run: withdraw any claim of a directional association for a pair if that pair's 95% interval includes zero."
}
out="result-v2.json"
with open(out,"w") as f: json.dump(result,f,indent=2); f.write("\n")
print(json.dumps(result,ensure_ascii=False,indent=2))