"""Random sample of Hacker News comments per half-year via the official HN Firebase API
(item ids are sequential, so uniform random ids inside a date range = a random sample of that range).
Measures em dashes and ChatGPT-style vocabulary per 10,000 words. Comment text is not stored, only counts.
Output: data/fb_sample.csv (one row per half-year) + data/fb_words.csv (per-word hits per half-year)."""
import json, urllib.request, datetime, random, re, html, sys, csv, collections
from concurrent.futures import ThreadPoolExecutor

UA = {"User-Agent": "tensorust-research (tensorust.com)"}
def item(i):
    for k in range(3):
        try:
            return json.load(urllib.request.urlopen(urllib.request.Request(f"https://hacker-news.firebaseio.com/v0/item/{i}.json", headers=UA), timeout=30))
        except Exception as e:
            if "429" in str(e) or "403" in str(e):
                print("blocked", e, file=sys.stderr); raise SystemExit(2)
    return None

MAX = json.load(urllib.request.urlopen("https://hacker-news.firebaseio.com/v0/maxitem.json"))
def t_of(i):
    for d in range(0, 50):
        it = item(i + d)
        if it and it.get("time"): return it["time"]
    return None
def id_at(ts):
    lo, hi = 1, MAX
    while hi - lo > 200:
        mid = (lo + hi) // 2
        if t_of(mid) < ts: lo = mid
        else: hi = mid
    return lo

VOCAB = ["delve", "delves", "delving", "tapestry", "testament", "underscores", "showcasing", "intricate", "meticulous",
         "meticulously", "pivotal", "realm", "seamless", "seamlessly", "robust", "leverage", "nuanced", "multifaceted",
         "commendable", "boasts", "garnered", "notably", "landscape", "embark", "holistic", "vibrant", "bustling",
         "foster", "streamline", "crucial", "comprehensive", "elevate", "unleash", "synergy", "paradigm"]
CONTROL = ["because", "actually", "probably", "however", "honestly"]
WRE = re.compile(r"\b(" + "|".join(VOCAB + CONTROL) + r")\b", re.I)
PHRASES = {"you're absolutely right": re.compile(r"you['’]re absolutely right", re.I),
           "not just x, it's y": re.compile(r"\b(?:isn['’]t|is not|not) just\b[^.?!]{1,60}[,;—-]\s*(?:it['’]s|but)\b", re.I)}

N = int(sys.argv[1]) if len(sys.argv) > 1 else 1500
random.seed(20261024)
periods = []
for y in range(2019, 2027):
    for h in (1, 2):
        a = datetime.datetime(y, 1 if h == 1 else 7, 1, tzinfo=datetime.timezone.utc)
        b = datetime.datetime(y, 7, 1, tzinfo=datetime.timezone.utc) if h == 1 else datetime.datetime(y + 1, 1, 1, tzinfo=datetime.timezone.utc)
        b = min(b, datetime.datetime(2026, 10, 24, tzinfo=datetime.timezone.utc))
        if a < b: periods.append((f"{y}-H{h}", int(a.timestamp()), int(b.timestamp())))

rows, wrows = [], []
with ThreadPoolExecutor(12) as ex:
    for name, a, b in periods:
        ia, ib = id_at(a), id_at(b)
        ids = random.sample(range(ia, ib), int(N * 1.4))
        its = list(ex.map(item, ids))
        com = [x for x in its if x and x.get("type") == "comment" and not x.get("deleted") and not x.get("dead") and x.get("text")][:N]
        words = em = em_c = dd = 0; wc = collections.Counter(); pc = collections.Counter()
        for x in com:
            t = html.unescape(re.sub(r"<[^>]+>", " ", x["text"]))
            words += len(t.split()); e = t.count("—"); em += e; em_c += e > 0
            dd += len(re.findall(r"\s--\s", t))
            for m in WRE.findall(t): wc[m.lower()] += 1
            for k, r in PHRASES.items(): pc[k] += len(r.findall(t))
        ai = sum(wc[w] for w in VOCAB)
        rows.append([name, ia, ib, len(com), words, em, em_c, dd, ai, pc["you're absolutely right"], pc["not just x, it's y"]])
        for w in VOCAB + CONTROL: wrows.append([name, w, wc[w]])
        print(rows[-1], file=sys.stderr)
with open("data/fb_sample.csv", "w", newline="") as f:
    c = csv.writer(f); c.writerow(["half_year", "first_item_id", "last_item_id", "comments_sampled", "words", "em_dashes", "comments_with_em_dash", "spaced_double_hyphens", "ai_vocab_hits", "youre_absolutely_right", "not_just_x_its_y"]); c.writerows(rows)
with open("data/fb_words.csv", "w", newline="") as f:
    c = csv.writer(f); c.writerow(["half_year", "word", "hits"]); c.writerows(wrows)
print("done", file=sys.stderr)