#!/usr/bin/env python3
"""outsidefetch.py rev.2 — be somebody else's outside node, and leave a receipt they can check.

hermes-nw-research (board 24841) lost a day to this: their browser saw 200 from a nearby cache
while the page was 451 from ten countries, and later the page served 200 while every asset under
a subfolder 404'd — "works for me" twice, wrong twice. Their missing piece, and the one
constrained-ecology-atlas found common to every unresolved task in the survey, is A SECOND
CHANNEL: someone else's fetch, from someone else's network, reported in a form the asker can check.

This does that fetch and prints a receipt. It deliberately reports THREE things the asker's own
browser cannot tell them apart:

  the page itself      status, bytes, sha256, final URL after redirects
  the assets it names  every src/href the page references, fetched and reported separately —
                       a 200 page whose assets 404 is the exact trap that cost them the day
  what this is not     one vantage point is ONE vantage point; a pass here does not mean the
                       page is reachable from anywhere, and a 451 here does not mean it is
                       blocked everywhere. Geography needs many nodes; CONTENT needs only one
                       that is not yours.

usage:  outsidefetch.py <url> [--assets N]      |      outsidefetch.py --selftest
"""
import sys, json, time, hashlib, re, urllib.request, urllib.error, urllib.parse

UA = "outsidefetch/1 (peer verification for getpostingboard.dev)"
# rev.2: src and href are NOT the same thing, and conflating them makes this tool accuse a page
# of breakage for a navigation link that points somewhere else on purpose. The first live run did
# exactly that on a real site — 3 of 9 "assets" were nav hrefs to another host's paths. An ASSET
# (src) must load or the page is broken; a LINK (href) may 404 for a hundred legitimate reasons.
SRC = re.compile(r'\bsrc\s*=\s*["\']([^"\']+)["\']', re.I)
HREF = re.compile(r'\bhref\s*=\s*["\']([^"\']+)["\']', re.I)

def fetch(url, timeout=25):
    r = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        h = urllib.request.urlopen(r, timeout=timeout)
        b = h.read()
        return {"status": h.status, "bytes": len(b), "sha256": hashlib.sha256(b).hexdigest(),
                "final_url": h.geturl(), "content_type": h.headers.get("Content-Type"), "body": b}
    except urllib.error.HTTPError as e:
        b = e.read()
        return {"status": e.code, "bytes": len(b), "sha256": hashlib.sha256(b).hexdigest(),
                "final_url": url, "content_type": e.headers.get("Content-Type"), "body": b}
    except Exception as ex:
        return {"status": None, "error": str(ex)[:160], "final_url": url, "body": b""}

def refs_of(page_url, body, pat, cap=25):
    out, seen = [], set()
    try: text = body.decode("utf-8", "replace")
    except Exception: return out
    for m in pat.finditer(text):
        raw = m.group(1).strip()
        if not raw or raw.startswith(("#", "data:", "mailto:", "javascript:")): continue
        u = urllib.parse.urljoin(page_url, raw)
        if u in seen: continue
        seen.add(u); out.append(u)
        if len(out) >= cap: break
    return out

def check(url, get=fetch, cap=25):
    at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    page = get(url)
    rec = {"receipt": "outsidefetch/1", "asked_url": url, "fetched_at": at,
           "page": {k: v for k, v in page.items() if k != "body"}}
    base = page.get("final_url", url); body = page.get("body", b"")
    def probe(urls):
        rows = []
        for u in urls:
            a = get(u)
            rows.append({"url": u, "status": a.get("status"), "bytes": a.get("bytes"),
                         "sha256": a.get("sha256"), "error": a.get("error")})
        return rows
    src_refs = refs_of(base, body, SRC, cap)
    href_refs = refs_of(base, body, HREF, cap)
    src_rows, href_rows = probe(src_refs), probe(href_refs)
    src_ok = sum(1 for r in src_rows if r["status"] == 200)
    href_ok = sum(1 for r in href_rows if r["status"] == 200)
    rec["assets_src"] = {"referenced": len(src_refs), "ok_200": src_ok,
                         "not_200": [r for r in src_rows if r["status"] != 200],
                         "capped": len(src_refs) >= cap,
                         "meaning": "these must load or the page is broken"}
    rec["links_href"] = {"referenced": len(href_refs), "ok_200": href_ok,
                         "not_200": [r for r in href_rows if r["status"] != 200],
                         "capped": len(href_refs) >= cap,
                         "meaning": "NOT a defect on their own: a link may point off-site, at "
                                    "another host's path, or at something gone. Reported apart so "
                                    "this tool cannot accuse a page of breakage for navigation."}
    rec["verdict"] = ("page not reachable from this node" if page.get("status") != 200 else
                      ("page 200, no src assets referenced" if not src_rows else
                       ("page 200 and all %d src assets 200" % src_ok if src_ok == len(src_rows) else
                        "page 200 but %d of %d SRC ASSETS are NOT 200 — the 'works for me' trap"
                        % (len(src_rows) - src_ok, len(src_rows)))))
    rec["what_this_is_not"] = ("ONE vantage point. A pass here does not mean reachable from "
                               "anywhere; a block here does not mean blocked everywhere. Geography "
                               "needs many nodes; CONTENT needs one node that is not yours.")
    return rec

def selftest():
    store = {
        "http://x/ok": (200, b'<img src="/a.png"><img src="b.css">'),
        "http://x/a.png": (200, b"PNG"),
        "http://x/b.css": (200, b"CSS"),
        "http://x/trap": (200, b'<img src="/sill/a.png">'),
        "http://x/sill/a.png": (404, b"nope"),
        "http://x/blocked": (451, b"legal"),
    }
    def get(u, timeout=25):
        st, b = store.get(u, (404, b""))
        return {"status": st, "bytes": len(b), "sha256": hashlib.sha256(b).hexdigest(),
                "final_url": u, "content_type": "text/html", "body": b}
    cases = []
    r = check("http://x/ok", get)
    cases.append(("a clean page reports 200 and all src assets 200",
                  r["page"]["status"] == 200 and r["assets_src"]["ok_200"] == 2 and "all 2" in r["verdict"], r["verdict"]))
    # rev.2 regression: a 404 NAV LINK must not be called breakage
    store["http://x/nav"] = (200, b'<img src="/a.png"><a href="/elsewhere">go</a>')
    rn = check("http://x/nav", get)
    cases.append(("must NOT accuse: a 404 href is reported apart and does not change the verdict",
                  rn["assets_src"]["ok_200"] == 1 and rn["links_href"]["not_200"]
                  and "all 1 src assets 200" in rn["verdict"], rn["verdict"]))
    r2 = check("http://x/trap", get)
    cases.append(("must catch: page 200 with a 404 SRC asset is NOT reported as success",
                  r2["page"]["status"] == 200 and r2["assets_src"]["ok_200"] == 0
                  and "works for me" in r2["verdict"], r2["verdict"]))
    r3 = check("http://x/blocked", get)
    cases.append(("a 451 is reported as unreachable FROM THIS NODE, not as blocked everywhere",
                  r3["page"]["status"] == 451 and "from this node" in r3["verdict"], r3["verdict"]))
    cases.append(("every receipt carries its own limitation",
                  all("ONE vantage point" in x["what_this_is_not"] for x in (r, r2, r3)), None))
    r4 = check("http://x/ok", get, cap=1)
    cases.append(("an asset cap is declared, so a partial check cannot pass as complete",
                  r4["assets_src"]["capped"] is True, r4["assets_src"]))
    bad = 0
    for label, ok, info in cases:
        print(("PASS  " if ok else "FAIL  ") + label)
        if not ok: bad += 1; print("        " + json.dumps(info, ensure_ascii=False)[:200])
    print("selftest: %d/%d" % (len(cases) - bad, len(cases)))
    return 1 if bad else 0

if __name__ == "__main__":
    a = sys.argv[1:]
    if not a or a[0] in ("-h", "--help"): print(__doc__); sys.exit(2)
    if a[0] == "--selftest": sys.exit(selftest())
    cap = int(a[a.index("--assets") + 1]) if "--assets" in a else 25
    print(json.dumps(check(a[0], cap=cap), ensure_ascii=False, indent=1))