import json, collections, sys
sys.path.insert(0, '/tmp/claude-1000/-var-www-html-peta/310517d2-8f28-442b-9907-0e3560add2cb/scratchpad/match')
from normalize import identity_numbers
D = '/var/www/html/peta/storage/app/propertylab-catalogue-match'
rc = {r['scheme_id']: r for r in json.load(open(f'{D}/realtycheck_schemes.json'))}
cat = json.load(open(f'{D}/catalogue_my_projects.json'))
out = json.load(open(f'{D}/match_v3.json'))
final = json.load(open(f'{D}/final.json'))
audit = json.load(open(f'{D}/review/_audit.json'))

def fnum(x):
    try:
        v = float(x); return v if v > 0 else None
    except (TypeError, ValueError):
        return None

# duplicates: pairs the matcher flagged as twins AND whose phase/section numbers agree; grouped transitively over those pairs only
nums = {}
def N(i):
    if i not in nums: nums[i] = identity_numbers(cat[i]['project_name'])
    return nums[i]
parent = {}
def find(x):
    while parent.get(x, x) != x: x = parent[x]
    return x
for o in out:
    if not o['top']: continue
    p0 = o['top'][0]['i']
    for c in o['top'][1:]:
        if c.get('twin') and N(c['i']) == N(p0):
            parent[find(c['i'])] = find(p0)
comp = collections.defaultdict(set)
for x in list(parent): comp[find(x)].add(x); comp[find(x)].add(find(x))
dup_list = sorted((sorted(v) for v in comp.values() if len(v) > 1), key=lambda g: (cat[g[0]]['state'] or '', cat[g[0]]['project_name']))

same = [f for f in final if f['status'].startswith('SAME')]
distinct = len({f['cat_i'] for f in same})
HR = {'Condo/Apartment', 'Serviced Apartment', 'Flat'}
pr, ps = [], []
for f in same:
    r = rc[f['scheme_id']]; p = cat[f['cat_i']]
    a, b = fnum(r['median_rm']), fnum(p['price_median'])
    if a and b: pr.append(a / b)
    if r['category'] in HR:
        a, b = fnum(r['reported_psf']), fnum(p['psf_median'])
        if a and b: ps.append(a / b)
within = lambda v, lo, hi: round(100 * sum(1 for x in v if lo <= x <= hi) / len(v)) if v else 0

A = audit.get('A_SAME', {}); nA = sum(A.values())
E4 = audit.get('E4_NOT_IN_CATALOGUE', {}); nE4 = sum(E4.values())
weak_groups = {t: audit.get(t, {}) for t in ('E1_DIFFERENT_PROPERTY_TYPE', 'E3_SAME_NAME_OTHER_PLACE', 'E5_WEAK')}
key2 = json.load(open(f'{D}/review/_key2.json'))
fin = {f['scheme_id']: f for f in final}
r2_recovered_same = sum(1 for k in key2.values() if not k['why'].startswith('A:') and fin[k['scheme_id']]['status'].startswith('SAME'))
r2_recovered_part = sum(1 for k in key2.values() if not k['why'].startswith('A:') and fin[k['scheme_id']]['status'].startswith('PART'))
r2_rule_matches = [k for k in key2.values() if k['why'].startswith('A:')]
r2_rejected = sum(1 for k in r2_rule_matches if fin[k['scheme_id']]['status'] == 'NOT IN CATALOGUE')
r2_unsure = sum(1 for k in r2_rule_matches if fin[k['scheme_id']]['status'] == 'NEEDS HUMAN CHECK')
review_n = sum(1 for f in final if f['method'].startswith('AI review'))
web_n = sum(1 for f in final if f.get('web'))
wg = ', '.join(f"{t.split('_', 1)[1].replace('_', ' ').lower()} {c.get('DIFFERENT', 0)}/{sum(c.values())}" for t, c in weak_groups.items())
trust = [
    f"Hand check by the analyst (read one by one): 170 random rule matches — 0 wrong; 40 random round-1 AI 'confident' verdicts — 0 wrong; "
    f"30 random round-2 AI 'confident' verdicts that overturned a rule rejection — 0 wrong; 25 random 'probable' verdicts — about 22 right, 3 doubtful.",
    f"Blind audit: {nA} random rule matches were re-judged by an AI reviewer who did not know they were decided — it agreed on {A.get('SAME', 0)}. "
    f"{nE4} random 'nothing similar in the catalogue' rejections — it agreed on {E4.get('DIFFERENT', 0)}.",
    f"The same audit showed three rejection groups were leaky ({wg} confirmed as different), so round 2 re-reviewed {len(key2):,} schemes in those groups "
    f"and in the riskiest rule matches: {r2_recovered_same} rejected schemes turned out to be SAME and {r2_recovered_part} PART OF; "
    f"of {len(r2_rule_matches)} risky rule matches, {r2_rejected} were overturned and {r2_unsure} sent to 'needs check'.",
    f"Prices corroborate the SAME matches: median sale price within ±15% for {within(pr, 0.85, 1.15)}% of the {len(pr):,} pairs where both sides have one; "
    f"for condos/apartments/flats PSF within ±10% for {within(ps, 0.9, 1.1)}% of {len(ps):,} pairs. Landed PSF is not comparable between the sources (different area basis) and was not used.",
    "Most 'NOT IN CATALOGUE' is expected: realtycheck covers every recorded scheme in Malaysia — small-town tamans, kampung, shops, industrial — while the catalogue is mostly residential and concentrated in the Klang Valley, Penang and Johor.",
]
method = [
    "Names normalised on both sides: JPPH abbreviations expanded (TMN→Taman, SG→Sungai, BKT→Bukit, KG→Kampung…), land-title references (TR / PL / PT / LOT numbers) removed, building-type words (Apartment, Condominium, Pangsapuri…) set aside, phase / precinct / section numbers compared separately.",
    "Candidates: every catalogue project sharing a distinctive name word or a near-spelling of one. Estate suffixes (Indah, Jaya, Baru, Permai…) and Kampung-vs-Taman count as different estates.",
    "Location: distance between the two map points, with an allowance that depends on type (about 300 m for a building, 1.5 km for a landed estate). A unique name in the state with matching prices overrides a bad coordinate.",
    "Property class compared (landed vs high-rise vs shop / industrial); median sale price (and PSF for high-rise) used as corroboration.",
    f"Catalogue duplicates (the same development listed twice) recognised and collapsed to one row; {len(dup_list):,} such groups listed on their own tab.",
    f"{review_n:,} schemes were judged one by one by AI reviewers ({web_n} of them with a web search) — the grey zone in round 1, the leaky groups in round 2 — plus the blind audit above.",
]
json.dump({'snapshot_collected': '2026-09-07', 'catalogue_my': len(cat), 'distinct_cat_same': distinct, 'dup_groups': len(dup_list),
           'dup_list': dup_list, 'trust_lines': trust, 'method_lines': method}, open(f'{D}/report_stats.json', 'w'))
print('dup groups', len(dup_list), '| distinct catalogue projects with SAME', distinct, '| price pairs', len(pr), within(pr, .85, 1.15), '% | psf pairs', len(ps), within(ps, .9, 1.1), '%')