#!/usr/bin/env python3
"""code_v14.py — apply the published v1.4 coding rules to the Stage A extraction.

Input:  extraction/<doc_id>.json (one per headline report; schema in EXTRACTION-SCHEMA.md)
Output: coded_v14.csv      one row per report, ten coded dimensions + the quote and page behind each
        rosters_v14.csv    one row per named member (working file; NOT published, see the codebook)
        roster-composition_v14.csv  per-report composition without names (this is the published table)
        citations_v14.csv  edge list: report -> institution/organization it cites
        coded_v14_report.txt  distributions and crosstabs, for the codebook

Every code is a deterministic function of extraction fields. There are no per-document
judgment calls in this script: where the extractor flagged an alternative reading, the rule
below decides. Re-running this script on the published extraction reproduces the CSVs exactly.

Run: python3 code_v14.py
"""
import json, csv, glob, re, os, collections

HERE = os.path.dirname(os.path.abspath(__file__))
os.chdir(HERE)

# ---------------------------------------------------------------- load
def _open_data(*names):
    """Published copies carry different filenames from the working folder; try both."""
    import os
    for n in names:
        if os.path.exists(n):
            return open(n)
    raise FileNotFoundError(names[0])


MAN = {m['doc_id']: m for m in json.load(open('extraction/manifest.json'))}
# v1.3 convening authority, read from landing pages and transmittals rather than the document.
# Used only as a fallback when the document itself does not name a convener.
# The v1.3 convening authority, read from landing pages rather than from the documents. It travels in
# the published inventory as `convening_authority_landing`, so this script runs from the published
# files alone; the frozen backup is only a fallback for the working folder.
def _landing():
    for f, col in (('ai-taskforce-inventory-v1.csv', 'convening_authority_landing'),
                   ('ai-taskforce-inventory-R1.csv', 'convening_authority_landing'),
                   ('ai-taskforce-inventory-R1.v13-backup.csv', 'convening_authority')):
        try:
            rows = list(csv.DictReader(open(f)))
        except FileNotFoundError:
            continue
        if rows and col in rows[0]:
            return {(r['institution'], r['url']): r[col] for r in rows if r['in_r1_census'] == 'TRUE'}
    return {}


LANDING = _landing()
DOCS = []
for f in sorted(glob.glob('extraction/*.json')):
    b = os.path.basename(f)
    if b.startswith('_') or b == 'manifest.json':
        continue
    DOCS.append(json.load(open(f)))
# Only headline rows are coded. A document moved to supplementary keeps its extraction file so the
# decision can be revisited, but it is not in the published coded table.
def _headline_urls():
    for f in ('ai-taskforce-inventory-v1.csv', 'ai-taskforce-inventory-R1.csv'):
        try:
            return {r['url'] for r in csv.DictReader(open(f)) if r['in_r1_census'] == 'TRUE'}
        except FileNotFoundError:
            continue
    return None


_HEAD = _headline_urls()
if _HEAD is not None:
    DOCS = [d for d in DOCS if MAN[d['doc_id']]['url'] in _HEAD]
DOCS.sort(key=lambda d: (MAN[d['doc_id']]['segment'], d['institution'], d['doc_id']))


def clip(x, n):
    x = re.sub(r'\s+', ' ', str(x or '')).strip()
    if len(x) <= n:
        return x
    cut = x[:n]
    return (cut[:cut.rfind(' ')] if ' ' in cut else cut).rstrip(' ,;:') + '...'


def sub(d, *path, default=''):
    x = d
    for k in path:
        if not isinstance(x, dict):
            return default
        x = x.get(k)
        if x is None:
            return default
    return x


# ---------------------------------------------------------------- rule 1: convening authority
# Taken from the document, not the landing page. `joint` is kept as one code because the
# alternative (splitting it) leaves cells of one and two; the pairing is carried in
# `joint_composition`, classified from the convener string by the keyword rules below,
# first match wins, in the order listed.
JOINT_RULES = [
    ('admin_and_senate',      r'senate|faculty council|faculty chair|university faculty'),
    ('provost_and_cio',       r'\bcio\b|vpit|information technolog|chief information'),
    ('provost_and_research',  r'\bvpr\b|vice president for research|research\b|\bogc\b|general counsel'),
    ('provost_and_operations',r'evpt|treasurer|chief operating|\bcoo\b|business officer|finance'),
    ('president_and_provost', r'president'),
]


def code_convener(d):
    role = sub(d, 'charge', 'convener_role', default='not_evidenced') or 'not_evidenced'
    source = 'document'
    if role in ('not_evidenced', ''):
        # the document does not name a convener: fall back to the landing page or transmittal
        land = LANDING.get((MAN[d['doc_id']]['institution'], MAN[d['doc_id']]['url']), '')
        if land and land != 'unknown':
            role, source = ({'joint_admin_senate': 'joint', 'system_senate': 'faculty_senate'}.get(land, land)), 'landing_page'
        else:
            role, source = 'unknown', 'none'
    printed = (sub(d, 'charge', 'convener_as_printed') or '') + ' ' + (sub(d, 'leadership', 'reports_to') or '')
    comp = ''
    if role == 'joint':
        for name, pat in JOINT_RULES:
            if re.search(pat, printed, re.I):
                comp = name
                break
        comp = comp or 'other_pair'
    return role, comp, printed.strip()[:300], source


# ---------------------------------------------------------------- rule 2: student role
# A printed roster is evidence of absence only when it is complete and its members are typed.
#   untyped : half or more of the roster entries carry no title, so category is unknown
#   partial : the roster is shorter than the membership total the report states, or names two
#             or fewer people (a signature block, not a membership list)
# Order: a named student member wins; a body that is itself a student body wins; then, if the
# roster is untyped or partial, the role the report states in prose; then absence.
def code_student(d):
    mem = d.get('membership') or {}
    roster = mem.get('roster') or []
    stated = mem.get('students_role') or 'not_evidenced'
    body = MAN[d['doc_id']].get('body_name') or ''      # body name only: a title about students is not a student body
    students = [r for r in roster if (r.get('category') or '') == 'student']
    voting = [r for r in students if (r.get('role') or 'member') in ('member', 'chair', 'co-chair')]
    if voting:
        return 'voting_member', len(students)
    if (re.search(r'\bstudent', body, re.I) and re.search(r'advisor|council|committee|group', body, re.I)
            and stated not in ('consulted_only', 'advisory', 'absent')):
        return 'voting_member', len(students)          # the body is itself a student body
    unknown = sum(1 for r in roster if (r.get('category') or 'unknown') == 'unknown')
    total = mem.get('total') if isinstance(mem.get('total'), int) else None
    untyped = bool(roster) and unknown >= len(roster) / 2
    partial = bool(roster) and ((total is not None and len(roster) < total) or len(roster) <= 2)
    if roster and not (untyped or partial):
        return ('consulted_only' if stated in ('consulted_only', 'advisory') else 'absent'), len(students)
    if stated == 'voting_member':
        return 'voting_member', len(students)
    if stated in ('consulted_only', 'advisory'):
        return 'consulted_only', len(students)
    if stated == 'absent':
        return 'absent', 0
    return 'not_evidenced', len(students)


# ---------------------------------------------------------------- rules 3-7, 9: direct maps
DIRECT = {
    # detector_stance is post-processed below: 'not_mentioned' splits into 'silent' (no quote in
    # the extraction) and 'mentions_no_position' (the extractor recorded a detector sentence but
    # the report takes no position on using them).
    'detector_stance':      ('detectors', 'position', {'not_mentioned': 'silent'}),
    'default_posture':      ('default_posture', 'value', {}),
    'syllabus_architecture':('syllabus', 'requirement', {'not_mentioned': 'silent', 'instructor_discretion_only': 'instructor_discretion'}),
    'standing_body_rec':    ('standing_body', 'proposed', {'not_evidenced': 'not_stated'}),
    'procurement_stance':   ('procurement_stance', 'value', {'not_evidenced': 'silent', 'against': 'against_licensing'}),
    'ai_self_disclosure':   ('ai_disclosure', 'report_used_ai', {}),
}


# ---------------------------------------------------------------- rule 8: seriousness (0-3)
# +1 if any recommendation names an owning unit or role
# +1 if any recommendation carries a deadline
# +1 if the report prints a dollar figure or an FTE ask
MONEY = re.compile(r'\$|\bfte\b|\bmillion\b|\bstaff line|\bfaculty line', re.I)


def code_seriousness(d):
    recs = d.get('recommendations') or []
    owners = sum(1 for r in recs if (r.get('owner_named') or 'none').strip().lower() not in ('', 'none', 'not_evidenced'))
    deadlines = sum(1 for r in recs if (r.get('deadline') or 'none').strip().lower() not in ('', 'none', 'not_evidenced'))
    res = sub(d, 'resources', 'dollar_amounts', default=[]) or []
    fte = sub(d, 'resources', 'fte_asks', default=[]) or []
    rec_money = sum(1 for r in recs if MONEY.search(str(r.get('resource') or '')))
    money = 1 if (res or [x for x in fte if str(x).strip()] or rec_money) else 0
    score = (1 if owners else 0) + (1 if deadlines else 0) + money
    return score, owners, deadlines, len(res), len([x for x in fte if str(x).strip()])


# ---------------------------------------------------------------- rule 10: citation normalization
# Free-text cited names are normalized to a canonical node, first match wins. Anything that
# matches no pattern keeps its printed name, trimmed. The map is published with the script so
# the network can be rebuilt or re-normalized by anyone.
CITE_MAP = [
    ('OpenAI', r'openai|chatgpt\b(?!.*polic)'), ('Microsoft', r'microsoft|copilot'), ('Google', r'\bgoogle\b|gemini|\bbard\b'),
    ('Anthropic', r'anthropic|claude'), ('Turnitin', r'turnitin'), ('Grammarly', r'grammarly'),
    ('National Science Foundation', r'\bnsf\b|national science foundation'),
    ('National Institutes of Health', r'\bnih\b|national institutes of health'),
    ('US Copyright Office', r'copyright office'), ('Pew Research Center', r'pew research'),
    ('World Economic Forum', r'world economic forum'), ('Springer Nature', r'springer|\bnature\b(?! of)'),
    ('Elsevier', r'elsevier'), ('Coursera', r'coursera'), ('LinkedIn Learning', r'linkedin learning'),
    ('EDUCAUSE', r'educause'), ('Ithaka S+R', r'ithaka'), ('AAC&U', r'aac&u|aacu|american association of colleges'),
    ('MLA-CCCC', r'\bmla\b|cccc'), ('UNESCO', r'unesco'), ('NIST', r'\bnist\b|national institute of standards'),
    ('US Dept of Education', r'department of education|office of educational technology|\bed\.gov'),
    ('White House / executive order', r'white house|executive order|\bostp\b|blueprint for an ai bill'),
    ('EU AI Act', r'\beu ai act\b|european union|european commission'),
    ('Chronicle of Higher Education', r'chronicle of higher'), ('Inside Higher Ed', r'inside higher ed'),
    ('Big Ten Academic Alliance', r'big ten'), ('AAU', r'\baau\b|association of american universities'),
    ('Ivy Plus / consortium', r'ivy plus|consortium of'),
    ('University of Michigan', r'\bmichigan\b(?!.*tech)'), ('Harvard University', r'harvard'),
    ('Stanford University', r'stanford'), ('Cornell University', r'cornell'), ('Yale University', r'\byale\b'),
    ('MIT', r'\bmit\b|massachusetts institute'), ('Princeton University', r'princeton'),
    ('Columbia University', r'columbia'), ('Duke University', r'\bduke\b'), ('Ohio State University', r'ohio state'),
    ('Arizona State University', r'arizona state|\basu\b'), ('Penn State', r'penn state|pennsylvania state'),
    ('University of Pennsylvania', r'university of pennsylvania|wharton|\bpenn\b'),
    ('Georgia Tech', r'georgia tech|georgia institute'), ('Purdue University', r'purdue'),
    ('Vanderbilt University', r'vanderbilt'), ('Carnegie Mellon University', r'carnegie mellon|\bcmu\b'),
    ('University of California (system or campus)', r'university of california|\buc\s|\bucla\b|berkeley|\bucsd\b|san diego|davis|irvine|santa barbara|santa cruz|riverside|merced|\bucsf\b'),
    ('University of Texas (system or campus)', r'university of texas|\but austin\b|\butsa\b|\butep\b'),
    ('Indiana University', r'indiana university'), ('University of Washington', r'university of washington'),
    ('University of Minnesota', r'university of minnesota'), ('University of Florida', r'university of florida'),
    ('Oxford University', r'oxford'), ('Cambridge University', r'cambridge'),
]


def split_cites(name):
    """One extraction entry can list several organizations ('NSF, NIH Bridge2AI, USDA NIFA').
    Split on semicolons and on commas that separate obvious organization names, then normalize each."""
    n = re.sub(r'\s+', ' ', (name or '')).strip()
    if not n:
        return []
    parts = [p.strip() for p in re.split(r'\s*;\s*', n) if p.strip()]
    out = []
    for p in parts:
        if re.search(r'\(', p) or len(p) < 30:
            out.append(p)
        else:                                   # long comma list of organizations, no parentheses
            out.extend([q.strip() for q in re.split(r',\s+(?=[A-Z])', p) if q.strip()])
    return out


# Strings that contain a university's name but do not refer to the university as a citing source.
CITE_NOT = re.compile(r'transfer agreement|assured admission|public universities. admissions', re.I)


# A university's name inside a string does not make the university the cited source. Three
# cases were being mapped to the institution through v1.4.2: a publication that carries the
# name (Harvard Business Review), a citation style that carries it (Harvard referencing), and
# an author's affiliation in a press interview. These are matched before CITE_MAP and keep
# their own identity. Where the real source is the venue, the venue is the node.
CITE_VENUE = [
    ('Harvard Business Review', r'harvard business review'),
    ('Citation style guides', r'(apa|mla|chicago)[^.]{0,40}\bharvard\b|harvard[^.]{0,20}(style|referencing)|'
                              r'harvard \((apa|mla)'),
    ('Wall Street Journal', r'wall street journal'),
]
# Longer institution names that contain a shorter one. Without these guards, University of
# British Columbia and University of Missouri-Columbia both counted as citations of Columbia.
CITE_GUARD = [
    (r'british columbia', 'University of British Columbia'),
    (r'missouri-columbia|missouri, columbia', 'University of Missouri'),
]


def norm_cite(name):
    n = re.sub(r'\s+', ' ', (name or '')).strip()
    if not n:
        return ''
    if CITE_NOT.search(n):
        return n[:80].strip()
    for canon, pat in CITE_VENUE:
        if re.search(pat, n, re.I):
            return canon
    for pat, canon in CITE_GUARD:
        if re.search(pat, n, re.I):
            return canon
    for canon, pat in CITE_MAP:
        if re.search(pat, n, re.I):
            return canon
    return n[:80].strip()


# standing_body_evidence: does the recorded quote STATE the coded position, or is the
# code a reading of the document as a whole? Through v1.4.1 this field was set to
# 'quoted' whenever any quote existed on the field, without testing whether the quote
# spoke to the question, which made it meaningless. A quote counts as 'stated' only if
# it names a standing structure; a 'no' additionally needs language that declines,
# sunsets or defers to existing structure. Everything else is 'inferred'.
SB_STRUCT = re.compile(r'\b(committee|council|office|standing|ongoing|permanent|task ?force|'
                       r'working group|oversight|steering|governance|board|cent(?:er|re)|'
                       r'bod(?:y|ies)|institute|advisory|hub)\b', re.I)
#
# v1.4.2 replaced the original rule with a keyword test, and an audit then showed that the test
# still checks word presence rather than entailment: it read CU Denver's "designed to work within
# established policy" as a refusal, when that passage is about not duplicating policies and says
# nothing about creating a body. So for the reports coded `no` the verdict is no longer computed.
# All 19 were read in full and adjudicated by hand into declines / defers / silent, with the
# supporting passage and page recorded in standing_body_adjudication.json, published beside this
# script. That file is the authority. The keyword test now covers only the `yes_*` values, where
# a quote naming the proposed structure is sufficient evidence.
#   declines: considered a continuing structure and decided against it, or sunset an existing one
#   defers:   routed the work to existing structures, proposing nothing new and refusing nothing.
#             Compatibility with existing governance is not a refusal.
#   silent:   never addressed whether anything continues
# convener_level: the printed title does not say whether a charging office sits at the campus
# or the system. A chancellor runs the campus at UC Davis and the system in the Colorado State
# and California State systems. Coded from the extraction for all 96 and published beside this script, with the
# supporting quote and a confidence flag, in convener_level.json. scope_level (whose people the
# recommendations bind) is coded independently: a system office can charter a body that reports
# on one campus, and a campus body can write something a system later adopts.
_CLV = 'convener_level.json'
CLV = {c['doc_id']: c for c in json.load(open(_CLV))} if os.path.exists(_CLV) else {}
_ADJ = 'standing_body_adjudication.json'
SB_ADJ = {a['doc_id']: a for a in json.load(open(_ADJ))} if os.path.exists(_ADJ) else {}
SB_DISPOSITION = {'declines': 'stated', 'defers': 'inferred', 'silent': 'inferred'}


def sb_evidence(value, quote, doc_id=None):
    if value == 'no':
        return SB_DISPOSITION.get(SB_ADJ.get(doc_id, {}).get('verdict'), 'inferred')
    q = (quote or '').strip()
    return 'stated' if q and SB_STRUCT.search(q) else 'inferred'


def sb_disposition(value, doc_id=None):
    """For a `no`, what the report actually did. Empty for the other values."""
    if value != 'no':
        return ''
    return SB_ADJ.get(doc_id, {}).get('verdict', 'not_adjudicated')


# ---------------------------------------------------------------- build
CODED_COLS = ['doc_id', 'institution', 'segment', 'date_published', 'url',
              'convening_authority', 'joint_composition', 'convener_source', 'convener_as_printed',
              'student_role', 'students_n', 'roster_n', 'roster_type', 'faculty_n', 'staff_n', 'admin_n', 'external_n',
              'reports_to', 'implementation_owner', 'successor_body',
              'detector_stance', 'default_posture', 'syllabus_architecture', 'standing_body_rec', 'standing_body_evidence', 'standing_body_disposition',
              'convener_level', 'convener_level_confidence', 'scope_level',
              'procurement_stance', 'seriousness', 'owners_named_n', 'deadlines_n', 'dollar_figures_n', 'fte_asks_n',
              'ai_self_disclosure', 'recommendation_count', 'policies_named_n', 'citations_n', 'pages', 'read_coverage',
              'q_detector', 'p_detector', 'q_posture', 'p_posture', 'q_syllabus', 'p_syllabus',
              'q_standing_body', 'p_standing_body', 'q_procurement', 'p_procurement', 'q_ai_disclosure', 'p_ai_disclosure',
              'extractor_notes']

coded, rosters, cites = [], [], []
for d in DOCS:
    did = d['doc_id']; m = MAN[did]
    role, comp, printed, csource = code_convener(d)
    srole, sn = code_student(d)
    roster = sub(d, 'membership', 'roster', default=[]) or []
    cats = collections.Counter((r.get('category') or 'unknown') for r in roster)
    # A report that prints its co-chairs and nothing else has not printed a roster. The extraction
    # records roster_present from the document; entries under roster_present=false are leadership
    # listings (six reports) and count as such, not as rosters. (v1.5.1, after external review.)
    if not roster:
        rtype = 'none'
    elif sub(d, 'membership', 'roster_present') is False:
        rtype = 'leadership_only'
    else:
        rtype = 'full'
    ser, owners, deadlines, ndollar, nfte = code_seriousness(d)
    row = {
        'doc_id': did, 'institution': d.get('institution') or m['institution'], 'segment': m['segment'],
        'date_published': m['date_published'], 'url': m['url'],
        'convening_authority': role, 'joint_composition': comp, 'convener_source': csource, 'convener_as_printed': printed,
        'student_role': srole, 'students_n': sn, 'roster_n': len(roster), 'roster_type': rtype,
        'faculty_n': cats['faculty'], 'staff_n': cats['staff'], 'admin_n': cats['administrator'], 'external_n': cats['external'],
        'reports_to': clip(sub(d, 'leadership', 'reports_to'), 300),
        'implementation_owner': clip(sub(d, 'leadership', 'in_charge_of_implementation'), 300),
        'successor_body': clip(sub(d, 'leadership', 'successor_body'), 300),
        'seriousness': ser, 'owners_named_n': owners, 'deadlines_n': deadlines,
        'dollar_figures_n': ndollar, 'fte_asks_n': nfte,
        'recommendation_count': d.get('recommendation_count') or len(d.get('recommendations') or []),
        'policies_named_n': len(d.get('policies_to_amend') or []),
        'citations_n': len(d.get('external_citations') or []),
        'pages': d.get('pages') or '', 'read_coverage': str(d.get('read_coverage'))[:60],
        'extractor_notes': str(d.get('extractor_notes'))[:400],
    }
    for col, (a, b, remap) in DIRECT.items():
        v = sub(d, a, b, default='not_evidenced') or 'not_evidenced'
        row[col] = remap.get(v, v)
        short = col.split('_')[0]
        key = {'detector_stance': 'detector', 'default_posture': 'posture', 'syllabus_architecture': 'syllabus',
               'standing_body_rec': 'standing_body', 'procurement_stance': 'procurement', 'ai_self_disclosure': 'ai_disclosure'}[col]
        row['q_' + key] = str(sub(d, a, 'quote'))[:300]
        row['p_' + key] = str(sub(d, a, 'page'))[:20]
    if row['detector_stance'] == 'silent' and str(sub(d, 'detectors', 'quote')).strip():
        row['detector_stance'] = 'mentions_no_position'
    row['standing_body_evidence'] = sb_evidence(row['standing_body_rec'], str(sub(d, 'standing_body', 'quote')), did)
    row['standing_body_disposition'] = sb_disposition(row['standing_body_rec'], did)
    _c = CLV.get(did, {})
    row['convener_level'] = _c.get('convener_level', 'not_evidenced')
    row['convener_level_confidence'] = (_c.get('confidence') or {}).get('convener') or ''
    row['scope_level'] = _c.get('scope_level', 'not_evidenced')
    coded.append(row)
    for r in roster:
        rosters.append({'doc_id': did, 'institution': row['institution'], 'segment': m['segment'],
                        'name': r.get('name', ''), 'title': r.get('title', ''), 'unit': r.get('unit', ''),
                        'category': r.get('category', 'unknown'), 'role': r.get('role', 'member')})
    seen = set()
    for c in d.get('external_citations') or []:
        for piece in split_cites(c.get('cited', '')):
            node = norm_cite(piece)
            if not node or node in seen:
                continue
            seen.add(node)
            cites.append({'doc_id': did, 'institution': row['institution'], 'segment': m['segment'],
                          'cited_raw': str(c.get('cited', ''))[:120], 'cited_node': node,
                          'context': str(c.get('context', ''))[:120], 'page': str(c.get('page', ''))[:20]})


# A citation entry that names a person and their title can split into a bare personal name and a
# stray title fragment. Those are not sources: drop any node that matches a name on one of the
# rosters, or that is only a job title. Works cited by author name are kept, since a citation to a
# published work is not a person's contact record.
_people = {r['name'] for r in rosters if len(r['name']) > 6}
_titlish = re.compile(r'^(associate |assistant |vice |deputy |interim )?(provost|dean|chair|director|president|chancellor)\b[^,]*\)?$', re.I)
cites = [c for c in cites if c['cited_node'] not in _people and c['cited_node'] != '[name withheld]'
         and not _titlish.match(c['cited_node'])]


def write(path, cols, rows):
    tmp = path + '.tmp'
    with open(tmp, 'w', newline='', encoding='utf-8') as f:
        w = csv.DictWriter(f, fieldnames=cols); w.writeheader(); w.writerows(rows)
    os.replace(tmp, path)
    print(f"{path}: {len(rows)} rows")


write('coded_v14.csv', CODED_COLS, coded)
write('rosters_v14.csv', ['doc_id', 'institution', 'segment', 'name', 'title', 'unit', 'category', 'role'], rosters)
write('citations_v14.csv', ['doc_id', 'institution', 'segment', 'cited_raw', 'cited_node', 'context', 'page'], cites)

# Committee composition without names: this is the roster table the site publishes. rosters_v14.csv,
# which carries the names, stays in the working folder and is not published (see the codebook, Privacy).
comp = collections.defaultdict(collections.Counter)
for r in rosters:
    comp[r['doc_id']][r['category']] += 1
    if r['role'] in ('chair', 'co-chair'): comp[r['doc_id']]['_chairs'] += 1
    if r['role'] == 'ex_officio': comp[r['doc_id']]['_ex'] += 1
crows = []
_rtype = {r['doc_id']: r['roster_type'] for r in coded}
for did, c in sorted(comp.items(), key=lambda kv: MAN[kv[0]]['institution']):
    m = MAN[did]
    crows.append({'doc_id': did, 'institution': m['institution'], 'segment': m['segment'], 'report_title': m['report_title'],
                  'roster_type': _rtype[did],
                  'members_total': sum(v for k, v in c.items() if not k.startswith('_')),
                  'faculty': c['faculty'], 'administrator': c['administrator'], 'staff': c['staff'], 'student': c['student'],
                  'external': c['external'], 'title_not_printed': c['unknown'],
                  'chairs_and_co_chairs': c['_chairs'], 'ex_officio': c['_ex']})
write('roster-composition_v14.csv', ['doc_id', 'institution', 'segment', 'report_title', 'roster_type', 'members_total', 'faculty',
      'administrator', 'staff', 'student', 'external', 'title_not_printed', 'chairs_and_co_chairs', 'ex_officio'], crows)

# ---------------------------------------------------------------- report
out = []
def p(s=''):
    out.append(str(s)); print(s)

def dist(col):
    return collections.Counter(r[col] for r in coded).most_common()

p(f"v1.4 coding over {len(coded)} headline reports\n")
for col in ['convening_authority', 'joint_composition', 'convener_source', 'student_role', 'detector_stance', 'default_posture',
            'syllabus_architecture', 'standing_body_rec', 'procurement_stance', 'seriousness', 'ai_self_disclosure']:
    p(f"{col}: {dict(dist(col))}")
p()
def xt(a, b):
    t = collections.defaultdict(collections.Counter)
    for r in coded:
        t[r[a]][r[b]] += 1
    p(f"--- {b} by {a}")
    for k in sorted(t):
        p(f"  {k:20} {dict(t[k])}")
xt('convening_authority', 'standing_body_rec')
xt('convening_authority', 'detector_stance')
xt('segment', 'student_role')
for r in coded: r['_year'] = (r['date_published'] or 'unknown')[:4]
xt('_year', 'procurement_stance')
xt('_year', 'standing_body_rec')
p()
v13 = dict(LANDING)
# v1.3 used a slightly different vocabulary: joint_admin_senate for any joint body, no separate
# chancellor code, "unknown" for not stated. Differences that are only vocabulary are reported
# separately from differences that change the fact.
def same(old, new, comp):
    if old == new:
        return True
    if old == 'joint_admin_senate' and new == 'joint' and comp == 'admin_and_senate':
        return True
    if {old, new} == {'president', 'chancellor'}:
        return True
    return False
vocab, subst = [], []
for r in coded:
    o = v13.get((r['institution'], r['url']), '?')
    if o == '?' or o == r['convening_authority']:
        continue
    (vocab if same(o, r['convening_authority'], r['joint_composition']) else subst).append(
        (r['institution'], o, r['convening_authority'], r['joint_composition']))
p(f"\nconvening authority, v1.3 (landing page) vs v1.4 (read from the document):")
p(f"  vocabulary only: {len(vocab)}   changed by reading the document: {len(subst)}   unchanged: {len(coded) - len(vocab) - len(subst)}")
for a, b, c, comp in subst:
    p(f"  {a[:40]:40} {b:20} -> {c}{(' (' + comp + ')') if comp else ''}")
cn = collections.Counter(c['cited_node'] for c in cites)
p(f"citation nodes: {len(cn)}; edges: {len(cites)}")
p("most cited: " + ', '.join(f"{k} {v}" for k, v in cn.most_common(20)))
rc = collections.Counter(r['category'] for r in rosters)
p(f"\nroster rows: {len(rosters)} across {len({r['doc_id'] for r in rosters})} reports; categories: {dict(rc)}")
open('coded_v14_report.txt', 'w').write('\n'.join(out))