"""Keep posts where bot/fake/spam/porn/crypto words sit near follow*, or that talk about removing/softblocking followers,
and the poster speaks about themself. Bot-labelled authors dropped. Writes candidates.json + cand.txt."""
import json, re
d = json.load(open("posts.json", encoding="utf-8"))
W = r"(bots?|botted|fake|spam\w*|porn\w*|crypto|scam\w*|ghost|inactive)"
F = r"(follow\w*)"
NEAR = re.compile(W + r"\W+(?:\w+\W+){0,5}?" + F + "|" + F + r"\W+(?:\w+\W+){0,5}?" + W, re.I)
REM = re.compile(r"(soft-?block\w*|remove\w* (a |my |the )?follow\w*|purg\w+ (my )?follow\w*|mass[- ]?block\w*|block ?list)", re.I)
MINE = re.compile(r"\b(my|me|i|i'm|i've|im|our|we)\b", re.I)
c = [p for p in d["posts"] if (NEAR.search(p["text"]) or REM.search(p["text"])) and MINE.search(p["text"]) and not p["bot"]]
c.sort(key=lambda p: p["time"])
json.dump(c, open("candidates.json", "w", encoding="utf-8"), ensure_ascii=False, indent=1)
with open("cand.txt", "w", encoding="utf-8") as f:
    for i, p in enumerate(c):
        f.write(f"[{i}] {p['handle']} {p['time'][:16]} r={p['is_reply']}\n{p['text'].replace(chr(10),' ')[:330]}\n\n")
print(len(d["posts"]), "->", len(c))
import collections
print("oldest per query (cap check):")
by = collections.defaultdict(list)
for p in d["posts"]:
    for s in p["src"]: by[s].append(p["time"])
cap = {q: min(t)[:16] for q, t in by.items()}
for q, t in sorted(cap.items(), key=lambda x: x[1], reverse=True)[:12]: print(" ", t, q)
json.dump(cap, open("query_oldest.json", "w"), indent=1)