#!/usr/bin/env python3
"""Recall benchmark: does a crawl output surface the known reports?
Usage: bench.py out1.json [out2.json ...]
Held-out set = the 69 documents at 55 institutions that were known before the v1.1 crawl re-run, frozen in
bench_holdout.json and flagged as bench_holdout=TRUE in the inventory CSV. Freezing the list matters: the
v1.4 coding pass rewrote coding_basis on every row, which is what the held-out set used to be derived from.
Set BENCH_ALL=1 to score against every headline row instead."""
import os
import json,csv,sys,re,collections,difflib,urllib.parse as up
def _open_data(*names):
    """Published copies carry different filenames from the working folder; try both."""
    import os
    for n in names:
        if os.path.exists(n):
            return open(n)
    raise FileNotFoundError(names[0])


AL=json.load(_open_data('name_aliases.json','data/name_aliases.json'))
R={}
for f in sys.argv[1:]: R.update(json.load(open(f)))
byname={}
for v in R.values(): byname.setdefault(v['name'],v)  # first occurrence wins
for v in R.values():
    if v.get('ai_candidates') or v.get('url_log'): byname[v['name']]=v
inv=list(csv.DictReader(_open_data('ai-taskforce-inventory-R1.csv','ai-taskforce-inventory-v1.csv')))
def norm(u):
    p=up.urlparse(u.strip().lower().split('#')[0]); return p.netloc.replace('www.','')+p.path.rstrip('/')
def base(u): return up.unquote(up.urlparse(u.lower()).path.rstrip('/').split('/')[-1])
def driveid(u):
    m=re.search(r'/d/([A-Za-z0-9_-]{20,})',u); return m.group(1) if m else None
def match(u,c):
    if norm(c)==norm(u): return True
    d=driveid(u)
    if d and d in c: return True
    b,bc=base(u),base(c)
    if len(b)>10 and difflib.SequenceMatcher(None,b,bc).ratio()>=0.85: return True
    return False
known=collections.defaultdict(list)
for r in inv:
    if (os.environ.get('BENCH_ALL') and r['in_r1_census']=='TRUE') or (not os.environ.get('BENCH_ALL') and r.get('bench_holdout')=='TRUE'): known[AL.get(r['institution'],r['institution'])].append(r['url'])
hit=[];miss=[]
for inst,urls in known.items():
    v=byname.get(inst)
    if not v: continue
    cand=[c['url'] for c in v.get('ai_candidates',[])]; visited=list(v.get('url_log',{}))
    found=[any(match(u,c) for c in cand) or any(norm(c)==norm(u) for c in visited) for u in urls]
    (hit if any(found) else miss).append((inst,urls,found,len(cand),v.get('html_pages_read',v.get('pages','')),{k:x for k,x in v.get('http_outcomes',{}).items() if k in('403','429')}))
n=len(hit)+len(miss)
print(f"institutions evaluated {n}; RECALL institution-level {len(hit)}/{n} = {len(hit)/n:.2f}")
tot=sum(len(u) for _,u,*_ in hit+miss); fnd=sum(sum(f) for _,_,f,*_ in hit+miss)
print(f"RECALL document-level {fnd}/{tot} = {fnd/tot:.2f}")
print("MISSES:")
for inst,urls,f,nc,rd,oc in miss:
    print(f" {inst[:42]:42} cands={nc:2} read={rd:3} {oc}")
    for u in urls: print('    ',u[:120])
print("PARTIAL:")
for inst,urls,f,nc,rd,oc in hit:
    for u,ok in zip(urls,f):
        if not ok: print(f" {inst[:35]:35} {u[:115]}")