"""Merge rule tiers + AI reviews (round 1 incl. blind audit, round 2) + manual overrides into one verdict per scheme."""
import json, glob, collections, os
D = '/var/www/html/peta/storage/app/propertylab-catalogue-match'
R = f'{D}/review'
rc = {r['scheme_id']: r for r in json.load(open(f'{D}/realtycheck_schemes.json'))}
cat = json.load(open(f'{D}/catalogue_my_projects.json'))
out = json.load(open(f'{D}/match_v3.json'))
overrides = json.load(open(f'{R}/_overrides.json')) if os.path.exists(f'{R}/_overrides.json') else {}

def load(pattern, key):
    got, bad = {}, []
    for fn in sorted(glob.glob(f'{R}/{pattern}')):
        for ln, line in enumerate(open(fn), 1):
            if not line.strip(): continue
            v = json.loads(line)
            if v.get('verdict') not in ('SAME', 'PART_OF', 'DIFFERENT', 'UNSURE') or v['case'] not in key:
                bad.append(f'{os.path.basename(fn)}:{ln}'); continue
            got[v['case']] = v
    return got, bad

key1, key2 = json.load(open(f'{R}/_key.json')), json.load(open(f'{R}/_key2.json'))
ai1, bad1 = load('batch_*_out.jsonl', key1)
ai2, bad2 = load('r2_batch_*_out.jsonl', key2)
print(f'round 1: {len(ai1)}/{len(key1)} verdicts ({len(bad1)} bad) | round 2: {len(ai2)}/{len(key2)} verdicts ({len(bad2)} bad)')
review = {}
for cid, k in key1.items():
    if cid in ai1: review[k['scheme_id']] = (cid, k, ai1[cid], 'AI review' + (' (blind audit)' if k['audit'] else ''))
for cid, k in key2.items():
    if cid in ai2: review[k['scheme_id']] = (cid, k, ai2[cid], 'AI review (round 2)')

RULE = {'A_SAME': 'SAME - confident', 'E1_DIFFERENT_PROPERTY_TYPE': 'NOT SAME - other property type',
        'E2_DIFFERENT_PHASE': 'NOT SAME - other phase', 'E3_SAME_NAME_OTHER_PLACE': 'NOT IN CATALOGUE',
        'E4_NOT_IN_CATALOGUE': 'NOT IN CATALOGUE', 'E5_WEAK': 'NOT IN CATALOGUE'}
audit = collections.defaultdict(collections.Counter)
final = []
for o in out:
    sid = o['scheme_id']; tier = o['tier']
    best = o['top'][0] if o['top'] else None
    rec = {'scheme_id': sid, 'tier': tier, 'best_i': best['i'] if best else None, 'best': best,
           'twins': [c['i'] for c in o['top'][1:] if c.get('twin')] if o['top'] else [],
           'top_all': [{'i': c['i'], 'dist_m': c['dist_m']} for c in o['top']],
           'status': RULE.get(tier, 'NEEDS HUMAN CHECK'), 'cat_i': best['i'] if tier == 'A_SAME' else None,
           'method': 'rules', 'confidence': 'high' if tier in RULE else None, 'reason': o['why'], 'web': False}
    if tier in ('E1_DIFFERENT_PROPERTY_TYPE', 'E2_DIFFERENT_PHASE') and best: rec['cat_i'] = best['i']
    if sid in review:
        cid, k, v, method = review[sid]
        if k.get('audit'): audit[tier][v['verdict']] += 1
        c = v.get('cand')
        ci = k['cand_i'][c] if isinstance(c, int) and 0 <= c < len(k['cand_i']) else None
        rec.update(method=method, confidence=v.get('confidence'), reason=v.get('reason', ''), web=bool(v.get('web')), case=cid)
        if v['verdict'] == 'SAME' and ci is not None:
            rec.update(status={'high': 'SAME - confident', 'medium': 'SAME - probable'}.get(v.get('confidence'), 'NEEDS HUMAN CHECK'), cat_i=ci)
        elif v['verdict'] == 'PART_OF' and ci is not None:
            rec.update(status='PART OF (phase / block / sub-estate)', cat_i=ci)
        elif v['verdict'] == 'DIFFERENT':
            keep = RULE.get(tier) if tier in ('E1_DIFFERENT_PROPERTY_TYPE', 'E2_DIFFERENT_PHASE') else 'NOT IN CATALOGUE'
            rec.update(status=keep, cat_i=rec['cat_i'] if keep != 'NOT IN CATALOGUE' else None)
        else:
            rec.update(status='NEEDS HUMAN CHECK', cat_i=ci)
    if sid in overrides:
        ov = overrides[sid]
        rec.update(status=ov['status'], cat_i=ov.get('cat_i'), method='manual review', reason=ov['reason'], confidence=ov.get('confidence', 'high'))
    final.append(rec)

print('\nBlind audit — AI verdict on schemes the rules had already decided:')
for t, c in sorted(audit.items()):
    n = sum(c.values()); agree = c['SAME'] if t == 'A_SAME' else c['DIFFERENT']
    print(f'  {t:28s} n={n:3d}  agrees {agree:3d} ({round(100 * agree / n)}%)  {dict(c)}')
print('\nFinal status:')
for s, n in collections.Counter(f['status'] for f in final).most_common(): print(f'  {s:40s} {n:6d}')
print('\nDecided by:', dict(collections.Counter(f['method'] for f in final)))
json.dump(final, open(f'{D}/final.json', 'w'))
json.dump({t: dict(c) for t, c in audit.items()}, open(f'{R}/_audit.json', 'w'))