#!/usr/bin/env python3
"""Chosen Module 16 fixtures, exact lexical counts and explicitly specified scores.
Python standard library only. No network, persistent state, timing or user logs.
"""
import argparse
from collections import Counter, defaultdict
import json
import math
from pathlib import Path
import re

K1, B = 1.2, 0.75

def load_corpus():
    return json.loads(Path(__file__).with_name('m16-corpus.json').read_text())

def tokenize(text):
    return re.findall(r'[a-z0-9]+', text.lower())

def statistics(documents):
    counts = {d['id']: Counter(tokenize(d['text'])) for d in documents}
    if len(counts) != len(documents):
        raise ValueError('duplicate document id')
    df = Counter(t for c in counts.values() for t in c)
    lengths = {key: sum(c.values()) for key, c in counts.items()}
    return counts, df, lengths

def postings(documents=None):
    documents = load_corpus() if documents is None else documents
    if len({d['id'] for d in documents}) != len(documents):
        raise ValueError('duplicate document id')
    result = defaultdict(list)
    for d in sorted(documents, key=lambda d:d['id']):
        positions = defaultdict(list)
        for position, term in enumerate(tokenize(d['text'])):
            positions[term].append(position)
        for term, places in positions.items():
            result[term].append({'id':d['id'], 'tf':len(places), 'positions':places})
    return dict(result)

def retrieve(query, mode='or', allowed=None, documents=None):
    documents = load_corpus() if documents is None else documents
    if mode not in ('and','or'):
        raise ValueError('unknown Boolean mode')
    terms = sorted(set(tokenize(query)))
    if not terms:
        return []
    index = postings(documents)
    sets = [{p['id'] for p in index.get(t,[])} for t in terms]
    ids = set.intersection(*sets) if mode == 'and' else set.union(*sets)
    if allowed is not None:
        ids &= set(allowed)
    return sorted(ids)

def coverage_tiers(query, allowed=None, limit=None, documents=None):
    """Count distinct matched terms, retain scope, then sort and limit."""
    if limit is not None and (type(limit) is not int or limit < 0):
        raise ValueError('limit must be a nonnegative integer')
    terms = set(tokenize(query))
    if not terms:
        return []
    index = postings(documents)
    scope = None if allowed is None else set(allowed)
    counts = Counter(p['id'] for term in terms for p in index.get(term, [])
                     if scope is None or p['id'] in scope)
    ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
    return ordered if limit is None else ordered[:limit]

def coverage_demo():
    query = 'database index maps rows'
    allowed = {'d1', 'd2', 'd4', 'd5', 'd8'}
    print(f'query={query!r} distinct terms={len(set(tokenize(query)))}')
    print('all coverage tiers:', coverage_tiers(query, allowed))
    print('limit 2:', coverage_tiers(query, allowed, limit=2))

def tfidf_scores(query, documents=None):
    documents = load_corpus() if documents is None else documents
    counts, df, lengths = statistics(documents)
    n = len(documents)
    idf = {t:math.log(n/f) for t,f in df.items()}
    def vector(c):
        return {t:(1+math.log(f))*idf[t] for t,f in c.items() if t in idf}
    def norm(v):
        return math.sqrt(sum(w*w for w in v.values()))
    qv = vector(Counter(tokenize(query)))
    qnorm = norm(qv)
    result=[]
    if not qnorm:
        return result
    for doc_id,c in counts.items():
        dv=vector(c); dnorm=norm(dv)
        dot=sum(weight*dv.get(t,0) for t,weight in qv.items())
        score=dot/(qnorm*dnorm) if dnorm else 0
        if score>0:
            result.append({'id':doc_id,'score':score,'dot':dot,'document_norm':dnorm,'query_norm':qnorm,'weights':dv})
    return sorted(result,key=lambda row:(-row['score'],row['id']))

def bm25_scores(query, documents=None, stats_documents=None):
    documents = load_corpus() if documents is None else documents
    stats_documents = documents if stats_documents is None else stats_documents
    counts, _, lengths = statistics(documents)
    _, df, stat_lengths = statistics(stats_documents)
    n=len(stats_documents); avgdl=sum(stat_lengths.values())/n if n else 0
    if not avgdl:
        return []
    terms=sorted(set(tokenize(query)) & set(df))
    result=[]
    for doc_id,c in counts.items():
        parts={}
        for t in terms:
            tf=c[t]
            idf=math.log(1+(n-df[t]+.5)/(df[t]+.5))
            factor=(tf*(K1+1))/(tf+K1*(1-B+B*lengths[doc_id]/avgdl)) if tf else 0
            parts[t]={'tf':tf,'df':df[t],'idf':idf,'factor':factor,'score':idf*factor}
        score=sum(p['score'] for p in parts.values())
        if score>0:
            result.append({'id':doc_id,'score':score,'length':lengths[doc_id],'parts':parts})
    return sorted(result,key=lambda row:(-row['score'],row['id']))

def phrase(query, documents=None):
    documents=load_corpus() if documents is None else documents
    q=tokenize(query)
    if not q:
        return []
    result=[]
    for d in documents:
        ts=tokenize(d['text'])
        if any(ts[i:i+len(q)]==q for i in range(len(ts)-len(q)+1)):
            result.append(d['id'])
    return sorted(result)

def fmt(rows):
    return ' '.join(f"{r['id']}:{r['score']:.6f}" for r in rows) or '(empty)'

def retrieval_demo():
    docs=load_corpus(); tokens=sum(len(tokenize(d['text'])) for d in docs)
    print(f'documents={len(docs)} tokens={tokens} vocabulary={len(postings(docs))}')
    print('exact id d5:', next(d['text'] for d in docs if d['id']=='d5'))
    for query in ['cache copies','caches','']:
        print(f'query={query!r} terms={tokenize(query)} OR={retrieve(query)} AND={retrieve(query,"and")}')
    query='cache copies'; matched=retrieve(query)
    print('nonmatches:', [d['id'] for d in docs if d['id'] not in matched])
    visits=0; scan_ids=[]
    for d in docs:
        found=False
        for token in tokenize(d['text']):
            visits+=1
            if token in set(tokenize(query)):
                found=True
        if found: scan_ids.append(d['id'])
    idx=postings(docs); terms=set(tokenize(query))
    looked_up=[entry for term in terms for entry in idx.get(term,[])]
    if scan_ids != matched or matched != sorted({entry['id'] for entry in looked_up}):
        raise RuntimeError('scan and lookup disagree')
    print(f'scan: documents={len(docs)} token_occurrences={visits}; prebuilt lookup: terms={len(terms)} postings={len(looked_up)}')
    if len(docs) != 8 or tokens != 50:
        raise RuntimeError('unexpected corpus')

def postings_demo():
    idx=postings()
    for term in ['cache','copies','to','a','index']:
        print(term, json.dumps(idx[term],separators=(',',':')))
    print('cache AND copies:', retrieve('cache copies','and'))
    print('cache OR copies:', retrieve('cache copies'))
    print('phrase stores copies:',phrase('stores copies'))
    print('phrase copies stores:',phrase('copies stores'))

def boolean_demo():
    allowed={'d1','d2','d4','d5','d8'}
    for query in ['cache stores','cache index','queue cache','']:
        strict=retrieve(query,'and',allowed); broad=retrieve(query,'or',allowed)
        fallback=[i for i in broad if i not in strict]
        print(f'{query!r}: AND={strict} OR={broad} tiers={[(i,0) for i in strict]+[(i,1) for i in fallback]}')
    print('all tiers retain allowed ids:', sorted(allowed))

def tfidf_demo():
    docs=load_corpus(); _,df,lengths=statistics(docs)
    print('N=8; query=cache index; logarithm=natural; normalization=cosine')
    print('df(cache)=2 df(index)=2 idf=',f'{math.log(4):.6f}')
    for r in tfidf_scores('cache index'):
        print(f"{r['id']}: query_norm={r['query_norm']:.6f} document_norm={r['document_norm']:.6f} dot={r['dot']:.6f} score={r['score']:.6f}")
    print('cache cache index:',fmt(tfidf_scores('cache cache index')))
    print('unknown quartz:',fmt(tfidf_scores('quartz')))
    print('empty query:',fmt(tfidf_scores('')))
    print('log_tf for 1,2,4:', ' '.join(f'{1+math.log(n):.6f}' for n in [1,2,4]))
    print('raw matching frequency:', ' '.join(f'{d["id"]}:{sum(Counter(tokenize(d["text"]))[t] for t in ["cache","index"])}' for d in docs if d['id'] in retrieve('cache index')))

def bm25_demo():
    docs=load_corpus(); _,_,lengths=statistics(docs)
    print('N=8 tokens=50 avgdl=6.25 k1=1.2 b=0.75; query=cache index')
    for r in bm25_scores('cache index'):
        present=[(t,p) for t,p in r['parts'].items() if p['tf']]
        t,p=present[0]
        print(f"{r['id']}: length={r['length']} term={t} tf={p['tf']} df={p['df']} idf={p['idf']:.6f} factor={p['factor']:.6f} score={r['score']:.6f}")
    print('repeat query unchanged:', bm25_scores('cache cache index')==bm25_scores('cache index'))
    print('TF-IDF order:', ' '.join(r['id'] for r in tfidf_scores('cache index')))
    print('BM25 order:', ' '.join(r['id'] for r in bm25_scores('cache index')))
    probes=[{'id':'p1','text':'cache'},{'id':'p2','text':'cache cache'},{'id':'p3','text':'cache stores copies close to readers'}]
    for r in sorted(bm25_scores('cache',probes,docs),key=lambda r:r['id']):
        print(f"probe {r['id']}: tf={r['parts']['cache']['tf']} length={r['length']} score={r['score']:.6f}")
    print('unknown/empty:',fmt(bm25_scores('quartz')),fmt(bm25_scores('')))

def pruning_demo():
    docs=load_corpus(); stop={'a','and','to'}
    pruned=[{**d,'text':' '.join(t for t in tokenize(d['text']) if t not in stop)} for d in docs]
    print('stop words:',sorted(stop))
    print('token occurrences:',sum(len(tokenize(d['text'])) for d in docs),'->',sum(len(tokenize(d['text'])) for d in pruned))
    print('phrase keys to rows: original=',phrase('keys to rows',docs),'pruned=',phrase('keys rows',pruned))
    print('phrase keys rows: original=',phrase('keys rows',docs),'pruned=',phrase('keys rows',pruned))
    champions={t:[r['id'] for r in bm25_scores(t)[:1]] for t in postings()}
    terms=['database','index']
    candidates=sorted({i for t in terms for i in champions[t]})
    complete=retrieve('database index','and')
    kept=sorted(set(complete)&set(candidates))
    print('champions database/index:',{t:champions[t] for t in terms})
    print('full OR candidates:',retrieve('database index'),'capped candidates:',candidates)
    print('full AND:',complete,'capped then AND:',kept,'missed:',sorted(set(complete)-set(kept)))

ALIASES={'db':'database','cache':['cache','replication']}

def understand(raw, allow_alias=False, allowed=None):
    if not isinstance(raw,str):
        raise ValueError('query must be text')
    # This tiny interface has no Boolean/phrase/field grammar. Refuse before tokenizing.
    if any(char in raw for char in '-:"') or any(t in {'not','and','or'} for t in tokenize(raw)):
        raise ValueError('unsupported query syntax')
    normalized=' '.join(raw.lower().split()); ts=tokenize(normalized)
    expanded=[]
    for t in ts:
        value=ALIASES.get(t,t) if allow_alias else t
        expanded.extend(value if isinstance(value,list) else [value])
    expanded=list(dict.fromkeys(expanded))
    return {'raw':raw,'normalized':normalized,'tokens':ts,'terms':expanded,'allowed':sorted(allowed) if allowed is not None else None,'ids':retrieve(' '.join(expanded),'or',allowed)}

def understanding_demo():
    allowed={'d1','d2','d4','d5','d8'}
    for raw,alias in [('  DB  INDEX ',False),('  DB  INDEX ',True),('cache',False),('cache',True),('quartz',True),('queue -retry',True)]:
        try:
            result=understand(raw,alias,allowed)
            print('aliases='+str(alias), json.dumps(result,separators=(',',':')))
        except ValueError as error:
            print(repr(raw)+': rejected: '+str(error))
    print('cache expansion without scope:',understand('cache',True)['ids'])

# Every impression tuple is one constructed exposure, not observed user traffic.
IMPRESSIONS=[('s1','cache','d1',1),('s1','cache','d4',2),('s2','cache','d4',1),('s2','cache','d1',2),('s3','cache','d1',1),('s3','cache','d4',2),('s4','cache','d4',1),('s4','cache','d1',2)]
CLICKS=[('e1','s1','d1'),('e1','s1','d1'),('e2','s2','d4'),('e3','s3','d1'),('e4','s4','d1'),('e5','missing','d1')]

def feedback_counts(impressions=IMPRESSIONS, clicks=CLICKS):
    exposed={(s,d):(q,p) for s,q,d,p in impressions}
    if len(exposed)!=len(impressions):
        raise ValueError('duplicate exposure identity')
    events={}; clicked=set(); duplicate=orphan=0
    for event,search,doc in clicks:
        if event in events:
            if events[event] != (search,doc):
                raise ValueError('event identity reused with changed payload')
            duplicate+=1; continue
        events[event]=(search,doc)
        if (search,doc) not in exposed:
            orphan+=1; continue
        clicked.add((search,doc))
    rows=[]
    for doc in sorted({d for _,d in exposed}):
        for position in [1,2]:
            keys=[key for key,value in exposed.items() if key[1]==doc and value[1]==position]
            count=sum(key in clicked for key in keys)
            rows.append({'id':doc,'rank':position,'exposed':len(keys),'clicked':count,'rate':count/len(keys) if keys else None})
    return {'rows':rows,'duplicate_events':duplicate,'orphan_events':orphan,'clicked_exposures':len(clicked),'exposures':len(exposed)}

def feedback_demo():
    counts=feedback_counts()
    for row in counts['rows']:
        print(f"{row['id']} rank={row['rank']}: clicked={row['clicked']} exposed={row['exposed']} rate={row['rate']:.2f}")
    print(f"events={len(CLICKS)} duplicate_events={counts['duplicate_events']} orphan_events={counts['orphan_events']}")
    print(f"clicked_exposures={counts['clicked_exposures']} exposures={counts['exposures']} rate={counts['clicked_exposures']/counts['exposures']:.2f}")

# Grades apply to these chosen information needs across ALL eight documents; others zero.
JUDGMENTS={'cache copies':{'d1':3,'d4':1,'d7':2},'index':{'d5':2,'d8':3},'quartz':{}}

def metrics(ranking, grades, k=3):
    if k<1 or len(ranking)!=len(set(ranking)):
        raise ValueError('positive k and distinct ranking required')
    top=ranking[:k]; relevant={i for i,g in grades.items() if g>0}
    hits=sum(i in relevant for i in top)
    dcg=sum((2**grades.get(doc,0)-1)/math.log2(rank+1) for rank,doc in enumerate(top,1))
    ideal=sorted(grades.values(),reverse=True)[:k]
    idcg=sum((2**g-1)/math.log2(rank+1) for rank,g in enumerate(ideal,1))
    first=next((rank for rank,doc in enumerate(top,1) if doc in relevant),None)
    return {'precision':hits/k,'recall':hits/len(relevant) if relevant else None,'rr':1/first if first else (0.0 if relevant else None),'dcg':dcg,'idcg':idcg,'ndcg':dcg/idcg if idcg else None,'hits':hits,'relevant':len(relevant)}

def evaluation_demo():
    sums={name:defaultdict(float) for name in ['id','bm25']}
    for query,grades in JUDGMENTS.items():
        for name,rank in [('id',retrieve(query)),('bm25',[r['id'] for r in bm25_scores(query)])]:
            values=metrics(rank,grades)
            rendered=' '.join(f'{key}={values[key]:.6f}' if values[key] is not None else key+'=NA' for key in ['precision','recall','rr','dcg','idcg','ndcg'])
            print(f'{name} {query!r} ranking={rank}: {rendered}')
            if grades:
                for key in ['precision','recall','rr','ndcg','hits','relevant']:
                    sums[name][key]+=values[key]
    for name,values in sums.items():
        print(f"{name} macro over 2 judged-positive queries: P@3={values['precision']/2:.6f} R@3={values['recall']/2:.6f} MRR@3={values['rr']/2:.6f} nDCG@3={values['ndcg']/2:.6f}; micro recall={values['hits']/values['relevant']:.6f}")
    print('empty answer for positive query:', json.dumps(metrics([],JUDGMENTS['index']),sort_keys=True))

MODES={'coverage':coverage_demo,'retrieval':retrieval_demo,'postings':postings_demo,'boolean':boolean_demo,'tfidf':tfidf_demo,'bm25':bm25_demo,'pruning':pruning_demo,'understanding':understanding_demo,'feedback':feedback_demo,'evaluation':evaluation_demo}
if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('mode',choices=MODES)
    MODES[parser.parse_args().mode]()
