"""Pull every tweet from one X account into JSONL + Markdown files.

Secrets are read from an env file and never printed. Any error text is
scrubbed of secret values before it reaches stdout or a log file.
"""

import json
import os
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path

import requests
from requests_oauthlib import OAuth1

ENV_PATH = Path(
    "/cursor/stores/bc-96ff1b3e-00b6-4da6-8a0d-5e72965f7e86/internal/secrets/x-api.env"
)
OUT_DIR = Path("/tmp/xpull/out")
USERNAME = "const_reborn"
API = "https://api.x.com/2"

TWEET_FIELDS = (
    "created_at,public_metrics,conversation_id,in_reply_to_user_id,"
    "referenced_tweets,entities,note_tweet,author_id,lang"
)
EXPANSIONS = "referenced_tweets.id"


def load_env(path: Path) -> dict:
    env = {}
    for line in path.read_text().splitlines():
        line = line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        k, v = line.split("=", 1)
        env[k.strip()] = v.strip().strip('"').strip("'")
    return env


ENV = load_env(ENV_PATH)
SECRET_VALUES = [v for k, v in ENV.items() if k != "X_ACCOUNT" and len(v) > 8]


def scrub(text: str) -> str:
    for v in SECRET_VALUES:
        text = text.replace(v, "[REDACTED]")
    return text


def log(msg: str) -> None:
    print(scrub(str(msg)), flush=True)


ERRORS: list[dict] = []


def record_error(step: str, resp: requests.Response | None, note: str = "") -> None:
    entry = {"step": step, "note": note}
    if resp is not None:
        entry["status"] = resp.status_code
        entry["body"] = scrub(resp.text[:1500])
    ERRORS.append(entry)
    log(f"[error] {step}: {entry.get('status')} {entry.get('body', note)}")


class Auth:
    def __init__(self, name: str, headers: dict | None = None, oauth1: OAuth1 | None = None):
        self.name = name
        self.headers = headers or {}
        self.oauth1 = oauth1

    def get(self, url: str, params: dict) -> requests.Response:
        return requests.get(url, params=params, headers=self.headers, auth=self.oauth1, timeout=60)


def bearer_auth() -> Auth:
    return Auth("bearer", headers={"Authorization": f"Bearer {ENV['X_BEARER_TOKEN']}"})


def oauth1_auth() -> Auth:
    return Auth(
        "oauth1",
        oauth1=OAuth1(
            ENV["X_API_KEY"],
            ENV["X_API_SECRET"],
            ENV["X_ACCESS_TOKEN"],
            ENV["X_ACCESS_TOKEN_SECRET"],
        ),
    )


def request_with_retries(auth: Auth, url: str, params: dict, step: str) -> requests.Response | None:
    """GET with 429 handling (sleep until x-rate-limit-reset) and 5xx backoff."""
    for attempt in range(8):
        try:
            resp = auth.get(url, params)
        except requests.RequestException as exc:
            log(f"[warn] {step}: network error {scrub(str(exc))}; retry {attempt}")
            time.sleep(5 * (attempt + 1))
            continue
        if resp.status_code == 429:
            reset = resp.headers.get("x-rate-limit-reset")
            remaining = int(resp.headers.get("x-rate-limit-remaining") or 0)
            if remaining > 0:
                # Window still has capacity: this is the per-second limit on search/all, so a short pause is enough.
                wait = 3 * (attempt + 1)
            else:
                wait = max(int(reset) - int(time.time()), 1) + 2 if reset else 60 * (attempt + 1)
                wait = min(wait, 16 * 60)
            log(f"[rate-limit] {step}: sleeping {wait}s (remaining="
                f"{resp.headers.get('x-rate-limit-remaining')}, limit={resp.headers.get('x-rate-limit-limit')})")
            time.sleep(wait)
            continue
        if resp.status_code >= 500:
            log(f"[warn] {step}: {resp.status_code}; retry {attempt}")
            time.sleep(10 * (attempt + 1))
            continue
        return resp
    return None


def lookup_user(auth: Auth) -> dict | None:
    resp = request_with_retries(
        auth,
        f"{API}/users/by/username/{USERNAME}",
        {"user.fields": "created_at,public_metrics,description,name,username,verified"},
        f"user lookup ({auth.name})",
    )
    if resp is None:
        record_error(f"user lookup ({auth.name})", None, "no response after retries")
        return None
    if resp.status_code != 200:
        record_error(f"user lookup ({auth.name})", resp)
        return None
    data = resp.json().get("data")
    log(f"[ok] user lookup via {auth.name}: id={data['id']} tweets={data['public_metrics']['tweet_count']}")
    return data


def paginate(auth: Auth, url: str, params: dict, step: str, out_path: Path, append: bool = False) -> tuple[int, str]:
    """Pull all pages; write raw tweet objects to out_path. Returns (count, status)."""
    count = 0
    seen: set[str] = set()
    token = None
    page = 0
    with out_path.open("a" if append else "w") as fh:
        while True:
            page += 1
            p = dict(params)
            if token:
                p["pagination_token" if "search" not in url else "next_token"] = token
            # Full-archive search allows 1 request per second.
            if "search/all" in url and page > 1:
                time.sleep(1.2)
            resp = request_with_retries(auth, url, p, f"{step} page {page}")
            if resp is None:
                record_error(f"{step} page {page}", None, "no response after retries")
                return count, "incomplete"
            if resp.status_code != 200:
                record_error(f"{step} page {page}", resp)
                return count, "blocked" if count == 0 else "incomplete"
            body = resp.json()
            includes = {t["id"]: t for t in body.get("includes", {}).get("tweets", [])}
            for tweet in body.get("data", []):
                if tweet["id"] in seen:
                    continue
                seen.add(tweet["id"])
                refs = []
                for ref in tweet.get("referenced_tweets", []) or []:
                    inc = includes.get(ref["id"])
                    if inc:
                        refs.append(inc)
                if refs:
                    tweet["_included_referenced_tweets"] = refs
                fh.write(dump_line(tweet) + "\n")
                count += 1
            fh.flush()
            meta = body.get("meta", {})
            log(f"[page {page}] {step}: +{meta.get('result_count', 0)} total={count} "
                f"remaining={resp.headers.get('x-rate-limit-remaining')}")
            token = meta.get("next_token")
            if not token:
                return count, "complete"


STORE_OUT = Path("/cursor/stores/bc-96ff1b3e-00b6-4da6-8a0d-5e72965f7e86/docs/sources/tweets")


def tweet_kind(t: dict) -> str:
    kinds = {r["type"] for r in t.get("referenced_tweets", []) or []}
    if "retweeted" in kinds:
        return "retweet"
    if "replied_to" in kinds:
        return "reply"
    if "quoted" in kinds:
        return "quote"
    return "original"


def full_text(t: dict) -> str:
    note = t.get("note_tweet") or {}
    return note.get("text") or t.get("text", "")


def dump_line(obj: dict) -> str:
    """JSON line with Unicode kept readable but U+2028/2029 escaped so line-based tools stay safe."""
    return json.dumps(obj, ensure_ascii=False).replace("\u2028", "\\u2028").replace("\u2029", "\\u2029")


def load_jsonl(path: Path) -> list[dict]:
    if not path.exists():
        return []
    rows, bad = [], 0
    # Split on \n only: tweet text can contain U+2028, which str.splitlines() would treat as a line break.
    for line in path.read_text().split("\n"):
        if not line.strip():
            continue
        try:
            rows.append(json.loads(line))
        except json.JSONDecodeError:
            bad += 1
    if bad:
        log(f"[warn] {path.name}: skipped {bad} unparseable line(s) (interrupted write)")
    return rows


def render(report: dict) -> dict:
    """Merge timeline + search results (dedupe by id), write JSONL, MD, and summary to the store."""
    merged: dict[str, dict] = {}
    for src in sorted(OUT_DIR.glob("*.jsonl")):
        for t in load_jsonl(src):
            merged.setdefault(t["id"], t)
    tweets = sorted(merged.values(), key=lambda t: t.get("created_at", ""), reverse=True)
    if not tweets:
        return {"written": False, "reason": "no tweets pulled"}

    STORE_OUT.mkdir(parents=True, exist_ok=True)
    with (STORE_OUT / f"{USERNAME}-tweets.jsonl").open("w") as fh:
        for t in tweets:
            fh.write(dump_line(t) + "\n")

    counts = {"total": len(tweets), "originals": 0, "replies": 0, "quotes": 0, "retweets": 0}
    for t in tweets:
        counts[{"original": "originals", "reply": "replies", "quote": "quotes", "retweet": "retweets"}[tweet_kind(t)]] += 1
    oldest, newest = tweets[-1]["created_at"][:10], tweets[0]["created_at"][:10]
    claimed = report["user"]["public_metrics"]["tweet_count"]
    sa = report.get("search_all") or {}
    archive_searched = sa.get("status") == "complete"
    complete = len(tweets) >= claimed - 25 or archive_searched
    if len(tweets) >= claimed - 25:
        status_line = "complete"
    elif archive_searched:
        status_line = (f"complete for every tweet the API still serves: {len(tweets)} of {claimed} counted on the profile; "
                       f"full-archive search covered account creation to now, so the {claimed - len(tweets)} missing are "
                       f"tweets X no longer returns (deleted, withheld, or replies to protected/suspended accounts)")
    else:
        status_line = "capped (timeline endpoint returns at most ~3,200 most recent)"
    marker = {"original": "ORIGINAL", "reply": "REPLY", "quote": "QUOTE", "retweet": "RETWEET"}

    def block(t: dict) -> str:
        m = t.get("public_metrics", {})
        url = f"https://x.com/{USERNAME}/status/{t['id']}"
        text = full_text(t).replace("\n", "\n> ")
        return (f"### {t['created_at'][:16].replace('T', ' ')} UTC · {marker[tweet_kind(t)]}\n\n"
                f"> {text}\n\n"
                f"Likes {m.get('like_count', 0)} · Reposts {m.get('retweet_count', 0)} · "
                f"Quotes {m.get('quote_count', 0)} · Replies {m.get('reply_count', 0)} · [link]({url})\n")

    lines = [f"# @{USERNAME} tweets", "",
             f"Newest first, grouped by month. {counts['total']} tweets, {oldest} to {newest}. "
             f"Pure retweets are listed at the end.", ""]
    month = None
    for t in tweets:
        if tweet_kind(t) == "retweet":
            continue
        m = t["created_at"][:7]
        if m != month:
            month = m
            lines += [f"## {month}", ""]
        lines.append(block(t))
    rts = [t for t in tweets if tweet_kind(t) == "retweet"]
    if rts:
        lines += ["## Retweets", ""]
        lines += [block(t) for t in rts]
    (STORE_OUT / f"{USERNAME}-tweets.md").write_text("\n".join(lines))

    top = sorted((t for t in tweets if tweet_kind(t) != "retweet"),
                 key=lambda t: t.get("public_metrics", {}).get("like_count", 0), reverse=True)[:30]
    s = [f"# @{USERNAME} tweet archive summary", "",
         f"- Account: {report['user']['name']} (@{report['user']['username']}), user id {report['user']['id']}",
         f"- Tweets claimed by profile: {claimed}",
         f"- Tweets pulled: {counts['total']} (originals {counts['originals']}, replies {counts['replies']}, "
         f"quotes {counts['quotes']}, retweets {counts['retweets']})",
         f"- Date range covered: {oldest} to {newest}",
         f"- Archive status: {status_line}",
         f"- Auth used: {report['auth_used']}; timeline: {report['timeline']}; search_all: {report['search_all']}",
         f"- Pulled on: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')} via X API v2 "
         f"(GET /2/users/:id/tweets + GET /2/tweets/search/all)",
         "", "## Top 30 by likes", ""]
    for i, t in enumerate(top, 1):
        m = t.get("public_metrics", {})
        snippet = full_text(t).replace("\n", " ")[:140]
        s.append(f"{i}. **{m.get('like_count', 0)} likes** · {t['created_at'][:10]} · {snippet} "
                 f"([link](https://x.com/{USERNAME}/status/{t['id']}))")
    if ERRORS:
        s += ["", "## API errors", ""] + [f"- {e['step']}: HTTP {e.get('status')} {e.get('body', e.get('note'))}" for e in ERRORS]
    (STORE_OUT / f"{USERNAME}-summary.md").write_text("\n".join(s) + "\n")
    return {"written": True, "counts": counts, "range": [oldest, newest], "complete": complete}


def read_usage() -> dict | None:
    """One call to /2/usage/tweets (app-only auth). Returns project usage or the scrubbed error."""
    resp = request_with_retries(bearer_auth(), f"{API}/usage/tweets", {}, "usage")
    if resp is None:
        return None
    if resp.status_code != 200:
        return {"status": resp.status_code, "body": scrub(resp.text[:500])}
    data = resp.json().get("data", {})
    return {k: data.get(k) for k in ("project_id", "project_usage", "project_cap", "cap_reset_day")}


def main() -> int:
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    report = {"username": USERNAME, "auth_used": None, "user": None, "timeline": None,
              "search_all": None, "errors": ERRORS}

    # OAuth 1.0a is only a fallback when user-context keys exist in the env file.
    auths = [bearer_auth()]
    if ENV.get("X_ACCESS_TOKEN") and ENV.get("X_ACCESS_TOKEN_SECRET"):
        auths.append(oauth1_auth())

    # --reuse-timeline: skip the user lookup and timeline pull, reuse out/timeline.jsonl + report.json (saves credits).
    reuse = "--reuse-timeline" in sys.argv and (OUT_DIR / "timeline.jsonl").exists() and (OUT_DIR / "report.json").exists()
    tl_path = OUT_DIR / "timeline.jsonl"
    if reuse:
        prev = json.loads((OUT_DIR / "report.json").read_text())
        user, report["auth_used"], report["timeline"] = prev["user"], prev["auth_used"], prev["timeline"]
        count = report["timeline"]["count"]
        log(f"[reuse] user id={user['id']} tweets={user['public_metrics']['tweet_count']}; timeline count={count}")
    else:
        user = None
        for auth in auths:
            user = lookup_user(auth)
            if user:
                report["auth_used"] = auth.name
                active = auth
                break
        if not user:
            log("[fatal] could not look up user with the available credentials")
            (OUT_DIR / "report.json").write_text(json.dumps(report, indent=2))
            return 1
        report["user"] = user

        tl_params = {
            "max_results": 100,
            "tweet.fields": TWEET_FIELDS,
            "expansions": EXPANSIONS,
        }
        count, status = paginate(active, f"{API}/users/{user['id']}/tweets", tl_params, "timeline", tl_path)
        if status == "blocked" and active.name == "bearer" and len(auths) > 1:
            log("[info] bearer blocked on timeline; retrying with OAuth 1.0a")
            active = auths[1]
            report["auth_used"] = active.name
            count, status = paginate(active, f"{API}/users/{user['id']}/tweets", tl_params, "timeline", tl_path)
        report["timeline"] = {"count": count, "status": status}
    report["user"] = user

    total_claimed = user["public_metrics"]["tweet_count"]
    if count < total_claimed - 50:
        # Full-archive search defaults to the last 30 days, so bound the window explicitly:
        # from account creation up to the oldest tweet the timeline already returned.
        # If a previous default-window (30-day) search exists, extend up to its oldest tweet instead,
        # because the timeline endpoint has been seen to skip tweets.
        recent = load_jsonl(OUT_DIR / "search_all_recent.jsonl")
        oldest_tl = min((t["created_at"] for t in (recent or load_jsonl(tl_path))), default=None)
        log(f"[info] timeline returned {count} of {total_claimed}; full-archive search "
            f"from {user['created_at']} to {oldest_tl}")
        sa_params = {
            "query": f"from:{USERNAME}",
            "max_results": 500,
            "start_time": user["created_at"],
            "tweet.fields": TWEET_FIELDS,
            "expansions": EXPANSIONS,
        }
        if oldest_tl:
            sa_params["end_time"] = oldest_tl
        sa_path = OUT_DIR / "search_all.jsonl"
        # --resume-search: continue an interrupted archive search from the oldest tweet already saved
        # (+1s so a tweet at the exact boundary second is not skipped; duplicates are removed at render).
        already = load_jsonl(sa_path) if "--resume-search" in sys.argv else []
        if already:
            oldest_saved = min(t["created_at"] for t in already)
            resume_from = datetime.fromisoformat(oldest_saved.replace("Z", "+00:00")) + timedelta(seconds=1)
            sa_params["end_time"] = resume_from.strftime("%Y-%m-%dT%H:%M:%SZ")
            log(f"[resume] {len(already)} archive tweets on disk; continuing before {sa_params['end_time']}")
        sa_count, sa_status = paginate(bearer_auth(), f"{API}/tweets/search/all", sa_params, "search_all", sa_path,
                                       append=bool(already))
        sa_count += len(already)
        if sa_status == "blocked" and ERRORS and ERRORS[-1].get("status") == 400:
            # A 400 means the parameters were rejected, not the tier; try the smaller page size once.
            sa_params["max_results"] = 100
            sa_count, sa_status = paginate(bearer_auth(), f"{API}/tweets/search/all", sa_params, "search_all(100)", sa_path)
        report["search_all"] = {"count": sa_count, "status": sa_status}
    else:
        log(f"[info] timeline returned {count} of {total_claimed}; full-archive search not needed")

    report["render"] = render(report)
    report["usage"] = read_usage()
    (OUT_DIR / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False))
    log(json.dumps({k: v for k, v in report.items() if k != "errors"}, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())