"""
scraper.py
Mengambil dan mengekstrak data mentah dari sebuah halaman website.
Menggunakan httpx + BeautifulSoup + lxml (ringan, tanpa browser headless)
supaya bisa jalan di hosting gratis seperti Render Free Tier.
"""
import re
import httpx
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 AIGO-Bot/1.0"
)

SOCIAL_DOMAINS = [
    "instagram.com", "tiktok.com", "facebook.com", "twitter.com", "x.com",
    "linkedin.com", "youtube.com", "wa.me", "whatsapp.com",
]

CTA_KEYWORDS = [
    "beli sekarang", "buy now", "hubungi kami", "contact us", "daftar",
    "sign up", "get started", "coba gratis", "free trial", "pesan sekarang",
    "order now", "konsultasi", "book now", "subscribe", "download",
    "shop now", "learn more", "pelajari lebih",
]

FAQ_KEYWORDS = ["faq", "pertanyaan", "frequently asked", "q&a", "tanya jawab"]
TESTIMONIAL_KEYWORDS = ["testimoni", "testimonial", "review", "ulasan", "klien kami", "our clients"]


class ScrapeError(Exception):
    pass


async def fetch_page(url: str) -> tuple[str, float]:
    """Ambil HTML mentah dari URL, kembalikan (html, waktu_load_detik)."""
    if not url.startswith("http://") and not url.startswith("https://"):
        url = "https://" + url

    parsed = urlparse(url)
    if not parsed.netloc:
        raise ScrapeError("That doesn't look like a valid URL.")

    headers = {"User-Agent": USER_AGENT}
    async with httpx.AsyncClient(follow_redirects=True, timeout=20.0, headers=headers) as client:
        try:
            import time
            start = time.perf_counter()
            resp = await client.get(url)
            elapsed = time.perf_counter() - start
        except httpx.RequestError as e:
            raise ScrapeError(f"Couldn't reach this website: {e}")

    if resp.status_code >= 400:
        raise ScrapeError(f"The website responded with status {resp.status_code}.")

    return resp.text, elapsed, str(resp.url)


def extract_data(html: str, final_url: str, load_time: float) -> dict:
    soup = BeautifulSoup(html, "lxml")
    text_lower = soup.get_text(" ", strip=True).lower()

    title = soup.title.string.strip() if soup.title and soup.title.string else ""

    meta_desc_tag = soup.find("meta", attrs={"name": "description"})
    meta_description = meta_desc_tag["content"].strip() if meta_desc_tag and meta_desc_tag.get("content") else ""

    meta_viewport = soup.find("meta", attrs={"name": "viewport"})
    has_viewport = meta_viewport is not None

    h1_tags = [h.get_text(strip=True) for h in soup.find_all("h1")]
    h2_tags = [h.get_text(strip=True) for h in soup.find_all("h2")]

    # Buttons / links yang berpotensi CTA
    clickable = soup.find_all(["a", "button"])
    cta_found = []
    for el in clickable:
        txt = el.get_text(strip=True).lower()
        if not txt:
            continue
        if any(k in txt for k in CTA_KEYWORDS):
            cta_found.append(el.get_text(strip=True))

    forms = soup.find_all("form")

    has_faq = bool(any(k in text_lower for k in FAQ_KEYWORDS))
    has_testimonial = bool(any(k in text_lower for k in TESTIMONIAL_KEYWORDS))

    # Kontak
    emails = list(set(re.findall(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", html)))
    whatsapp_links = [a["href"] for a in soup.find_all("a", href=True) if "wa.me" in a["href"] or "whatsapp" in a["href"]]
    phones = list(set(re.findall(r"(?:\+62|62|0)8[0-9]{8,11}", text_lower)))

    # Alamat kasar (kata kunci lokasi)
    has_address_hint = bool(re.search(r"\b(jl\.|jalan|street|no\.\s?\d+|kec\.|kelurahan)\b", text_lower))

    # Social links
    social_links = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        for domain in SOCIAL_DOMAINS:
            if domain in href:
                social_links.append(href)
                break
    social_links = list(set(social_links))

    # Images & alt text
    images = soup.find_all("img")
    images_total = len(images)
    images_missing_alt = len([img for img in images if not img.get("alt", "").strip()])

    # Nav structure
    nav = soup.find("nav")
    nav_links_count = len(nav.find_all("a")) if nav else len(soup.select("header a"))

    # SSL / https
    is_https = final_url.startswith("https://")

    # Word count (proxy for content depth)
    word_count = len(text_lower.split())

    return {
        "final_url": final_url,
        "load_time_seconds": round(load_time, 2),
        "is_https": is_https,
        "title": title,
        "title_length": len(title),
        "meta_description": meta_description,
        "meta_description_length": len(meta_description),
        "has_viewport": has_viewport,
        "h1_count": len(h1_tags),
        "h1_tags": h1_tags[:5],
        "h2_count": len(h2_tags),
        "cta_count": len(cta_found),
        "cta_examples": list(dict.fromkeys(cta_found))[:5],
        "form_count": len(forms),
        "has_faq": has_faq,
        "has_testimonial": has_testimonial,
        "emails": emails[:5],
        "whatsapp_links": whatsapp_links[:5],
        "phones": phones[:5],
        "has_address_hint": has_address_hint,
        "social_links": social_links,
        "images_total": images_total,
        "images_missing_alt": images_missing_alt,
        "nav_links_count": nav_links_count,
        "word_count": word_count,
    }


async def scrape_website(url: str) -> dict:
    html, load_time, final_url = await fetch_page(url)
    return extract_data(html, final_url, load_time)