#!/usr/bin/env python3 """crawl2 v0.5.0: batch domain crawler for university AI task force reports. Search-engine-free. v0.4 (2026-09-05), built after diagnosing every v0.2 miss against the known positives: * priority queue instead of FIFO: AI-tagged links first, then committee/task-force/report INDEX pages (they are archives), then other governance-host pages, then generic pages. Depth was never the constraint; the request budget was being spent on breadth before reaching indexes. * adaptive budget: 200 requests by default; if nothing AI-tagged has surfaced and high-priority pages remain queued, the budget extends to 350 for that institution only. * institutions may carry "alt_roots" (e.g. iu.edu + indiana.edu) and "extra_hosts" (e.g. twin-cities.umn.edu, manoa.hawaii.edu). The IPEDS web-address field points at sub-sites for several campuses; v0.2 treated that sub-site as the whole institution. * ~30 governance subdomain probes and ~40 path seeds (adds in., web., teaching., ctl., research., /acadaffairs, /offices/provost, /provost/ai, and the like: where v0.2's misses actually lived). * follows 308 redirects (urllib does not; Drexel was "blocked" because of this). * user-agent fallback: the declared research UA first; on 403 retry once with a browser UA and record ua_fallback=True for the host so the census can disclose it. * v0.4.1: link resolution uses the URL the server actually served (redirect target), so relative links on "/ai" -> "/ai/" pages resolve correctly (UTEP miss); curl fallback on TLS handshake failures (UTSA provost site); 429 backoff; per-section page cap (20) so an AI department's site cannot absorb the whole budget (Buffalo miss); in-scope hosts are followed when the LINK looks like a report even if the host name does not look like governance (MIT aiandeducation.); "ombud" and a few more probes. Per-HOST cap (40 pages) as well: an AI hub subdomain (ai.unc.edu, ai.psu.edu, ai.northeastern.edu) is all AI-tagged and was absorbing the budget; probed governance hosts now fetch first (priority -1) instead of queueing behind AI-tagged links. * v0.4.2 (after the first R2 pass): at R2 and smaller institutions governance usually lives under PATHS on the main host (/academics/provost, /about/leadership, /academic-affairs/faculty-senate) rather than on its own subdomain, and the homepage link to it says "Provost" or "Leadership", which matched neither the body vocabulary nor the index vocabulary, so 52 of 139 R2 crawls exhausted their queue after the seeds. Links whose path or anchor names a governance office are now followed from the homepage and from depth-1 pages; more path seeds. Also: a cookie jar (Fordham's CAS gateway redirect loops without one); "www." is only prefixed when it resolves (medicine.okstate.edu is a sub-site); when the homepage redirects to another registrable domain (uno.edu -> lsuneworleans.edu) that domain joins the scope; ".js" in SKIP was matching ".jsums.edu" and blanked Jackson State, extensions are now anchored. * v0.4.3: student-government subdomain probes (sga., asuc., studentgov., gpsa.), not yet run. * everything v0.3 added: successful-read count, offsite document capture (Drive/Box/repositories/ CDNs), HTML pages as report candidates, per-URL HTTP outcome. Usage: python3 crawl2.py list.json out.json [start] [end] """ import re, sys, json, time, socket, heapq, itertools, subprocess, urllib.parse as up, urllib.request as ur, collections, ssl, http.cookiejar from concurrent.futures import ThreadPoolExecutor ssl._create_default_https_context = ssl._create_unverified_context UA = "Mozilla/5.0 (compatible; higher-ed-report-inventory/0.4.2; academic research; kylesaunders.com/ai-taskforce-inventory)" UA2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" HDR2 = {"Accept": "text/html,application/xhtml+xml,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9"} DOC = re.compile(r"\.(pdf|docx?|pptx?)(\?|$)", re.I) # A hosted document carries no file extension, so DOC never fired on it and the +3 was lost. HOSTEDDOC = re.compile(r"(docs\.google\.com/(document|presentation|spreadsheets)|drive\.google\.com/file|\.sharepoint\.com/[^ ]*(:w:|:b:|:p:|Doc\.aspx)|box\.com/s/|dropbox\.com/s/)", re.I) AI = re.compile(r"\b(a\.?i\.?|artificial[- ]intelligence|genai|gen[- ]ai|generative[- ]ai|chatgpt|llms?|copilot)\b", re.I) BODY = re.compile(r"\b(task[- ]?forces?|taskforces?|working[- ]groups?|workgroups?|committees?|commissions?|advisory|steering|senates?|councils?|panels?)\b", re.I) OUT = re.compile(r"\b(reports?|recommendations?|findings?|final|white[- ]?papers?|charges?|frameworks?|strateg(?:y|ies)|plans?|guidelines?|guidance|principles?|roadmaps?)\b", re.I) GOVPATH = re.compile(r"(provost|academic[- ]?affairs|faculty|senate|governance|leadership|president|chancellor|administration|offices|policies|strategic|planning|teaching|ctl|cte|research|ai\b|artificial)", re.I) INDEX = re.compile(r"\b(committees?|task[- ]?forces?|working[- ]groups?|reports?|archives?|documents?|resolutions?|initiatives?|publications?|minutes)\b", re.I) GOVH = re.compile(r"(provost|senate|council|faculty|academic|president|chancellor|governance|^ai\.|\.ai\.|^ai$|^oit\.|^it\.|^cio\.|artificial|teaching|^ctl|^cte|research|policy|strateg|genai|^vp|^ovc|^ovp|^ufc)", re.I) ASSETHOST = re.compile(r"(jsdelivr|cloudflare|googleapis|gstatic|bootstrapcdn|fontawesome|unpkg\.com|jquery|typekit|doubleclick|google-analytics|googletagmanager|hotjar|addthis|recaptcha|youtube\.com|vimeo\.com|twitter\.com|facebook\.com|linkedin\.com|instagram)", re.I) NOHOST = re.compile(r"(source|news|giving|bookstore|catalog|static|shop|store|events|calendar|alumni|athletic|admission|housing|dining|librar|health|maps|jobs|career|blog|media|video|photo|cdn|^apps?\.|login|sso|^my\.|portal|^learn\.|canvas|^chat\.|communications|registrar|advising|international|^hr\.|engagement|secure\.|^m\.|mobile|search|analytics|^api\.|wiki|forum|mail)", re.I) SKIP = re.compile(r"(/news/|/events?/|calendar|profile|directory|jobs|giving|athletics|/tag/|/category/|login|\.(jpg|jpeg|png|gif|svg|css|js|mp4|ico|webmanifest|xml|json|zip)(\?|$)|/wp-json/|/feed/|\?share=|/page/\d|/people/|/staff/|/faculty-directory|/search|/story/|/stories/|/article/)", re.I) # extensions anchored: ".js" used to match ".jsums.edu" NEWSOK = re.compile(r"(/news/|/stories/|/story/|/article/|/posts?/)", re.I) # news pages allowed only when AI-tagged, max 15 SUBS = ["provost", "president", "chancellor", "facultysenate", "faculty-senate", "senate", "facultycouncil", "faculty", "academicaffairs", "academic-affairs", "academics", "ai", "oit", "it", "cio", "governance", "in", "web", "teaching", "ctl", "cte", "teach", "research", "vpaa", "policy", "policies", "genai", "strategicplan", "strategy", "ufc", "universitysenate", "univsenate", "facsen", "ovpr", "vpr", "ovcaa", "academic", "ai-hub", "innovation", "planning", "ombud", "ombuds", "leadership", "chancellor-office", "provost-office", "academicsenate", "facultyassembly", "assembly", "aiandeducation", "generativeai", "sga", "asuc", "studentgov", "studentgovernment", "asi", "gpsa", "gsa"] # student-government hosts, added v0.4.3 for the next crawl GOOD_SUBS = set(SUBS) # probed hosts pass the governance test by construction PATHS = ["/", "/provost", "/president", "/chancellor", "/facultysenate", "/faculty-senate", "/senate", "/academicaffairs", "/academic-affairs", "/acadaffairs", "/ai", "/artificial-intelligence", "/generative-ai", "/genai", "/committees", "/task-forces", "/taskforces", "/reports", "/offices/provost", "/office-of-the-provost", "/provost/ai", "/provost/initiatives", "/academics/provost", "/about/provost", "/about/leadership", "/administration", "/faculty-senate/committees", "/senate/committees", "/committees-and-task-forces", "/ai-task-force", "/ai-taskforce", "/generative-ai-task-force", "/artificial-intelligence-task-force", "/strategic-plan", "/policies", "/teaching", "/ctl", "/academic-affairs/ai", "/ai/reports", "/initiatives", "/university-senate", "/faculty-council", "/facultycouncil", "/academics/provost", "/academics/academic-affairs", "/academics/faculty-senate", "/about/leadership", "/about/administration", "/about/office-of-the-president", "/office-of-the-president", "/academic-senate", "/academicsenate", "/governance", "/faculty-governance", "/offices", "/administration/provost", "/provost-office", "/academics", "/about", "/faculty", "/faculty-staff", "/faculty-resources"] _ua_ok = {} # host -> "bot" | "browser" _dead = {} # host -> consecutive connection failures _jar = http.cookiejar.CookieJar() _opener = ur.build_opener(ur.HTTPCookieProcessor(_jar)) def _open(u, ua, extra, timeout): req = ur.Request(u, headers={"User-Agent": ua, **extra}) return _opener.open(req, timeout=timeout) def _curl(u, timeout): """Fallback for servers whose TLS stack rejects Python's handshake (observed: provost.utsa.edu).""" try: r = subprocess.run(["curl", "-sL", "-m", str(timeout + 5), "-A", UA2, "-w", "\n@@%{http_code} %{content_type} %{url_effective}", u], capture_output=True, timeout=timeout + 8) body, _, tail = r.stdout.decode("utf-8", "replace").rpartition("\n@@") code, ctype, final = (tail.split(" ", 2) + ["", ""])[:3] if not code.isdigit(): return None, "curl-fail", u if "html" not in ctype: return None, "non-html", final or u return body[:600_000], int(code), final or u except Exception as e: return None, "curl-" + type(e).__name__, u def get(u, timeout=7): """Return (html_or_None, outcome, final_url). Follows 308s. Retries with a browser UA on 403/429, backs off on 429, falls back to curl on TLS failures.""" host = up.urlparse(u).netloc if _dead.get(host, 0) >= 4: # host has timed out four times: stop waiting on it return None, "host-unreachable", u order = ["browser", "bot"] if _ua_ok.get(host) == "browser" else ["bot", "browser"] last = None for hops in range(4): for mode in order: ua, extra = (UA, {}) if mode == "bot" else (UA2, HDR2) try: r = _open(u, ua, extra, timeout) _ua_ok.setdefault(host, mode) final = r.geturl() or u if "html" not in r.headers.get("Content-Type", ""): return None, "non-html", final return r.read(600_000).decode("utf-8", "replace"), r.status, final except ur.HTTPError as e: last = e.code if e.code in (301, 302, 303, 307, 308) and e.headers.get("Location"): u = up.urljoin(u, e.headers["Location"]); host = up.urlparse(u).netloc break if e.code == 429 and mode == "bot": time.sleep(1.5); continue if e.code in (403, 503) and mode == "bot": continue return None, e.code, u except Exception as e: last = type(e).__name__ _dead[host] = _dead.get(host, 0) + 1 if "SSL" in repr(e) or "EOF" in repr(e): html, oc, final = _curl(u, timeout) if html is not None or oc == "non-html": _ua_ok.setdefault(host, "curl") return html, oc, final return None, last, u else: return None, last, u return None, last, u def links(html, base): """Return (url, anchor_text || ~300 chars of visible text before the link).""" out = [] for m in re.finditer(r'href=["\']([^"\'#]+)', html, re.I): h = m.group(1).strip() if h.startswith(("mailto:", "tel:", "javascript:")): continue u = up.urljoin(base, h) seg = html[m.end():m.end() + 400] seg = seg[seg.find(">") + 1:] if ">" in seg else seg t = re.sub(r"<[^>]+>", " ", seg.split("")[0])[:180] before = re.sub(r"<[^>]+>", " ", html[max(0, m.start() - 900):m.start()]) ctx = re.sub(r"\s+", " ", before).strip()[-300:] out.append((u, (re.sub(r"\s+", " ", t).strip() + " || " + ctx)[:500])) return out def page_body_signal(page_text): """True when a page both names a deliberative body and is about AI. Lends that body signal to the links leaving the page. Requiring the AI term as well keeps a generic "committees" nav item on an unrelated page from promoting everything it links. """ head = page_text[:3000] return bool(BODY.search(head) and AI.search(head)) def score(u, text, page_body=False): blob = up.unquote(u) + " " + text s = 0 has_ai = bool(AI.search(blob)) has_body = bool(BODY.search(blob)) or page_body has_out = bool(OUT.search(blob)) if has_ai: s += 3 if has_body: s += 3 if has_out: s += 2 if DOC.search(u) or HOSTEDDOC.search(u): s += 3 if has_ai and has_body and has_out: s += 4 return s class Scope: def __init__(self, roots, extra_hosts=()): self.roots = [r.lower() for r in roots] self.extra = {h.lower() for h in extra_hosts} def root_of(self, host): host = host.lower().split(":")[0] for r in self.roots: if host == r or host.endswith("." + r): return r return None def sub_of(self, host): r = self.root_of(host) if not r: return "" host = host.lower().split(":")[0] return host[:-len(r) - 1] if host != r else "" def bad_host(self, host): if host.lower() in self.extra: return False return bool(NOHOST.search(self.sub_of(host))) def good_host(self, host): if not self.root_of(host): return False if host.lower() in self.extra: return True s = self.sub_of(host) if s in ("", "www"): return False return (s in GOOD_SUBS or bool(GOVH.search(s))) and not self.bad_host(host) def crawl(root, alt_roots=(), extra_hosts=(), max_pages=200, extend_to=350, depth=3): sc = Scope([root, *alt_roots], extra_hosts) seen, hits, hosts, log = set(), {}, set(), {} seq = itertools.count() heap = [] def push(u, d, prio): heapq.heappush(heap, (prio, next(seq), u, d)) base = f"https://www.{root}" home, out0, hb = get(base) if not home: base = f"https://{root}" home, out0, hb = get(base) if home: fh = up.urlparse(hb).netloc if fh and not sc.root_of(fh): # homepage redirected to another domain newroot = ".".join(fh.split(".")[-2:]) if not fh.endswith(".edu") or fh.count(".") < 2 else ".".join(fh.split(".")[-2:]) sc.roots.append(newroot); alt_roots = list(alt_roots) + [newroot] for lu, lt in links(home, hb): h = up.urlparse(lu).netloc if sc.good_host(h): hosts.add(h) def probe(h): if h in hosts: return None try: socket.setdefaulttimeout(3); socket.gethostbyname(h) html, oc, _ = get(f"https://{h}/", timeout=5) return h if html or oc in (200, 403, 429) else None except Exception: return None cands = [f"{s}.{r}" for r in sc.roots for s in SUBS] + list(extra_hosts) with ThreadPoolExecutor(10) as ex: for h in ex.map(probe, cands): if h: hosts.add(h) for h in sorted(hosts)[:30]: push(f"https://{h}/", 0, -1) for p in PATHS: push(f"{base}{p}", 0, 1 if p != "/" else 2) for r in alt_roots: for p in PATHS[:12]: push(f"https://www.{r}{p}", 0, 1) def handle(item): prio, _, u, d = item html, outcome, final = get(u) return u, d, html, outcome, final n = ok = 0; newsn = 0 extended = False secn = {}; hostn = {} def section(x): p = up.urlparse(x); segs = [g for g in p.path.split('/') if g] return p.netloc + '/' + '/'.join(segs[:2]) while heap and n < max_pages: batch = [] while heap and len(batch) < 8: item = heapq.heappop(heap) u, d = item[2], item[3] if u in seen or d > depth or (SKIP.search(u) and not (NEWSOK.search(u) and item[0] == 0 and newsn < 15)): continue if NEWSOK.search(u): newsn += 1 seen.add(u); batch.append(item) if not batch: break with ThreadPoolExecutor(8) as ex: fetched = list(ex.map(handle, batch)) n += len(batch) for u, d, html, outcome, final in fetched: log[u] = outcome if not html: continue ok += 1 if final != u: seen.add(final); u = final sec = section(u) secn[sec] = secn.get(sec, 0) + 1 hn = up.urlparse(u).netloc hostn[hn] = hostn.get(hn, 0) + 1 m = re.search(r"(.*?)", html, re.I | re.S) page_title = re.sub(r"\s+", " ", m.group(1)).strip()[:120] if m else "" visible = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", re.sub( r"<(script|style).*?", " ", html, flags=re.I | re.S))) pbody = page_body_signal(page_title + " " + visible) if score(u, page_title) >= 12 and u not in hits: hits[u] = {"score": score(u, page_title), "anchor": page_title[:160], "found_on": u, "page_title": page_title, "html": True} for lu, lt in links(html, u): h = up.urlparse(lu).netloc if lu in seen: continue if ASSETHOST.search(lu): continue # CDNs and social embeds are not documents s_ = score(lu, lt, pbody) if not sc.root_of(h): platform = re.search(r"(drive\.google|docs\.google|box\.com|sharepoint|digitalcommons|scholarworks|repository|conservancy|cdn|dropbox|onedrive|issuu|adobe\.com|canva\.com)", lu, re.I) # hosting platforms need the usual bar; other offsite documents must carry all three signals if ((platform and s_ >= 8) or (DOC.search(lu) and s_ >= 12)) and lu not in hits: hits[lu] = {"score": s_, "anchor": lt.split(" || ")[0][:160], "context": lt.split(" || ")[-1][:200], "found_on": u, "page_title": page_title, "offsite": True} continue if sc.bad_host(h): continue if h not in hosts and (sc.good_host(h) or s_ >= 6): hosts.add(h); push(f"https://{h}/", d + 1, 1) if DOC.search(lu) or HOSTEDDOC.search(lu): if s_ >= 8 and lu not in hits: hits[lu] = {"score": s_, "anchor": lt.split(" || ")[0][:160], "context": lt.split(" || ")[-1][:200], "found_on": u, "page_title": page_title} continue anchor = lt.split(" || ")[0] path = up.urlparse(lu).path crowded = (secn.get(section(lu), 0) >= 20 or hostn.get(h, 0) >= 40) and s_ < 8 # section/host already ate its share if crowded: push(lu, d + 1, 3) elif s_ >= 6 or AI.search(up.unquote(lu) + " " + anchor): push(lu, d + 1, 0) elif INDEX.search(up.unquote(path) + " " + anchor) or BODY.search(up.unquote(path) + " " + anchor): push(lu, d + 1, 1) elif (sc.good_host(h) or s_ >= 6) and d < depth: push(lu, d + 1, 2) elif d <= 1 and GOVPATH.search(up.unquote(path) + " " + anchor): push(lu, d + 1, 2) # governance office reached by path, not subdomain elif d < depth - 1 and BODY.search(lt): push(lu, d + 1, 3) # adaptive budget: nothing AI-tagged yet, high-priority pages still queued if n >= max_pages and not extended: any_ai = any(AI.search(up.unquote(k) + " " + v.get("anchor", "") + " " + v.get("context", "") + " " + v.get("page_title", "")) for k, v in hits.items()) if not any_ai and any(p <= 1 for p, *_ in heap): max_pages, extended = extend_to, True ranked = sorted(hits.items(), key=lambda kv: -kv[1]["score"]) ua_fb = sorted(h for h, m in _ua_ok.items() if m in ("browser", "curl") and sc.root_of(h)) return [{"url": k, **v} for k, v in ranked], {"requests_attempted": n, "html_pages_read": ok, "budget_extended": extended, "outcomes": dict(collections.Counter(str(v) for v in log.values())), "url_log": log, "ua_fallback_hosts": ua_fb}, sorted(hosts) def run_one(inst): t0 = time.time() hits, m, hosts = crawl(inst["domain"], inst.get("alt_roots", []), inst.get("extra_hosts", [])) ai_hits = [h for h in hits if AI.search(up.unquote(h["url"]) + " " + h["anchor"] + " " + h.get("context", "") + " " + h.get("page_title", ""))] return {"name": inst["name"], "unitid": inst["unitid"], "domain": inst["domain"], "crawler": "0.4.2", "requests_attempted": m["requests_attempted"], "html_pages_read": m["html_pages_read"], "budget_extended": m["budget_extended"], "ua_fallback_hosts": m["ua_fallback_hosts"], "http_outcomes": m["outcomes"], "url_log": m["url_log"], "subdomains": hosts, "ai_candidates": ai_hits[:25], "other_candidate_document_links": len(hits) - len(ai_hits), "secs": round(time.time() - t0)} if __name__ == "__main__": lst = json.load(open(sys.argv[1])) outp = sys.argv[2] a = int(sys.argv[3]) if len(sys.argv) > 3 else 0 b = int(sys.argv[4]) if len(sys.argv) > 4 else len(lst) try: results = json.load(open(outp)) except Exception: results = {} for inst in lst[a:b]: key = inst["domain"] + "|" + str(inst["unitid"]) if key in results: continue r = run_one(inst) results[key] = r json.dump(results, open(outp, "w"), indent=1) print(f"{inst['name'][:40]:40} tried={r['requests_attempted']:3} read={r['html_pages_read']:3} ai={len(r['ai_candidates']):2} other={r['other_candidate_document_links']:3} ext={int(r['budget_extended'])} uafb={len(r['ua_fallback_hosts'])} {r['secs']}s", flush=True)