#!/usr/bin/env python3
"""
Import Properstar favorites via ListGlobally REST API.

Replaces the old DOM-scraping favorites_scraper.py with a reliable
API-based approach. Uses JWT token captured during Playwright login.

Usage:
    python3 import_favorites.py              # Import all new favorites
    python3 import_favorites.py --full       # Re-import all (update existing)
    python3 import_favorites.py --dry-run    # Show what would be imported
    python3 import_favorites.py --login      # Force fresh login first
"""
import argparse
import asyncio
import re
import sys
import time
from pathlib import Path

import requests

from store import load, persist, upsert, SCRIPT_DIR

FAVORITES_URL = 'https://listings-api.listglobally.com/api/v2/listings/favorites'
LISTING_URL = 'https://listings-api.listglobally.com/api/v2/listings'

API_HEADERS = {
    'accept': 'application/json',
    'accept-language': 'en',
    'referer': 'https://www.properstar.nl/',
}


TOKEN_PATH = SCRIPT_DIR / '.properstar_token'
COOKIES_PATH = SCRIPT_DIR / '.properstar_cookies.json'


def get_auth_token():
    """Read saved Bearer token."""
    if not TOKEN_PATH.exists():
        return None
    token = TOKEN_PATH.read_text().strip()
    if not token.startswith('eyJ'):
        return None
    return token


def save_auth_token(token):
    """Persist token to disk."""
    TOKEN_PATH.write_text(token)


def make_session(token):
    """Create a requests.Session with auth headers + browser cookies for session continuity."""
    session = requests.Session()
    session.headers.update({**API_HEADERS, 'authorization': f'Bearer {token}'})
    # Load cookies captured during Playwright login — server requires both JWT + session cookies.
    if COOKIES_PATH.exists():
        import json as _json
        for c in _json.loads(COOKIES_PATH.read_text()):
            session.cookies.set(c['name'], c['value'], domain=c.get('domain', ''))
    return session


def api_get(session, path, params=None):
    """Make authenticated GET request to ListGlobally API."""
    resp = session.get(path, params=params, timeout=15)
    if resp.status_code == 403:
        print("Auth token expired. Run with --login to refresh.")
        sys.exit(1)
    resp.raise_for_status()
    return resp.json()


def fetch_favorite_ids(session):
    """Get all favorite listing IDs in one call."""
    data = api_get(session, FAVORITES_URL, {
        'mode': 'ListingIds',
        'sort': '-LastActivityDate',
        'offset': 0,
        'limit': 500,
    })
    listings = data.get('listings', [])
    return [l['id'] for l in listings]


def fetch_listing_detail(session, listing_id):
    """Fetch full details for a single listing."""
    data = api_get(session, f'{LISTING_URL}/{listing_id}')
    return data.get('listing', data)


def extract_fields(listing):
    """Extract store-compatible fields from API listing data."""
    fields = {
        'source': 'properstar',
        'properstar_id': listing.get('id'),
        'title': listing.get('automaticTitle', ''),
        'property_type': (listing.get('type') or {}).get('id'),
        'sub_type': (listing.get('subType') or {}).get('id'),
        'status_api': (listing.get('state') or {}).get('id'),
        'favorited_at': listing.get('favoriteCreationDate'),
    }

    # Location
    loc = listing.get('location') or {}
    if loc.get('latitude'):
        fields['lat'] = loc['latitude']
        fields['lon'] = loc['longitude']
        fields['coord_source'] = 'properstar_api'
    fields['city'] = loc.get('city')
    fields['postal_code'] = loc.get('postcode')
    fields['country'] = loc.get('countryISO')

    # Price
    price_data = listing.get('price') or {}
    for pv in price_data.get('values', []):
        if pv.get('currencyId') == 'EUR':
            fields['price'] = int(pv['value'])
            break

    # Size
    area_data = listing.get('area') or {}
    for av in area_data.get('values', []):
        unit = (av.get('unit') or {}).get('id', '')
        if unit == 'SquareMeter':
            if av.get('living'):
                fields['building_size'] = av['living']
                fields['building_size_m2'] = av['living']
            if av.get('land'):
                fields['land_size'] = av['land']
                fields['land_size_m2'] = av['land']
                fields['land_size_verified'] = True
            break

    # Rooms
    num = listing.get('numberOf') or {}
    if num.get('bedrooms'):
        fields['bedrooms'] = int(num['bedrooms'])
    if num.get('rooms'):
        fields['rooms'] = int(num['rooms'])
    if num.get('floors'):
        fields['floors'] = int(num['floors'])

    # Description (first available language)
    descs = listing.get('descriptionFull') or []
    if descs:
        fields['description_lang'] = descs[0].get('language', 'unknown')

    # Energy rating
    for rating in (listing.get('ratings') or []):
        if rating.get('ratingType') == 'EnergyEu':
            fields['energy_class'] = rating.get('grade')
            break

    # Photos count
    pics = (listing.get('resources') or {}).get('pictures') or {}
    if pics.get('number'):
        fields['photo_count'] = pics['number']

    # Photos — thumbnail + full URL list
    items = pics.get('items', [])
    if items:
        fields['thumbnail'] = items[0].get('url', '')
        all_urls = [i.get('url', '') for i in items if i.get('url')]
        if all_urls:
            fields['photo_urls'] = all_urls

    return fields


async def enrich_stubs(stub_ids):
    """Visit each stub listing page in Playwright, intercept the API detail response.

    Checks availability (404/redirect = Removed) and captures full listing data.
    Returns dict: {id: listing_dict} for available ones, {id: None} for unavailable.
    """
    from playwright.async_api import async_playwright
    import re as _re

    results = {}
    intercepted = {}

    async def on_response(response):
        if 'listglobally.com/api/v2/listings' not in response.url:
            return
        # Match /listings/{id} with optional trailing query string or slash
        m = _re.search(r'/listings/(\d+)(?:[/?]|$)', response.url)
        if not m:
            return
        lid = int(m.group(1))
        if lid not in stub_ids:
            return
        if response.status == 200:
            try:
                body = await response.json()
                listing = body.get('listing', body)
                if listing.get('id'):
                    intercepted[lid] = listing
                    print(f"    [API] Captured {lid}")
            except Exception:
                pass
        elif response.status in (404, 410):
            intercepted[lid] = None  # confirmed unavailable via API
            print(f"    [API] {lid} → {response.status}")

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=False)
        context = await browser.new_context()
        page = await context.new_page()
        page.on('response', on_response)

        # Go to favorites page first — this proves auth AND fires the favorites API
        print("Opening Properstar favorites page to establish authenticated session...")
        auth_confirmed = []

        async def on_auth_confirm(response):
            if 'listglobally.com/api/v2/listings/favorites' in response.url and response.status == 200:
                auth_confirmed.append(True)

        page.on('response', on_auth_confirm)
        await page.goto("https://www.properstar.nl/favorites", timeout=120000)

        print("Log in if needed. Waiting for authenticated favorites API response (up to 5 min)...")
        for i in range(150):
            await asyncio.sleep(2)
            if auth_confirmed:
                print(f"Authenticated session confirmed. Enriching {len(stub_ids)} stubs...")
                break
            if i % 15 == 0 and i > 0:
                print(f"  Waiting for login... ({i * 2}s)")
        else:
            print("Timeout waiting for authenticated session.")
            await browser.close()
            return {}

        for lid in stub_ids:
            url = f"https://www.properstar.nl/listing/{lid}"
            print(f"  Checking {lid}...", end=' ', flush=True)
            try:
                resp = await page.goto(url, wait_until='networkidle', timeout=30000)

                # Check for redirect to homepage/search
                final_url = page.url
                if _re.search(r'properstar\.nl/(nl/)?$|/search|/zoeken', final_url):
                    print(f"Redirected → Removed")
                    results[lid] = None
                    continue

                # Check HTTP status
                if resp and resp.status in (404, 410):
                    print(f"{resp.status} → Removed")
                    results[lid] = None
                    continue

                # Wait up to 6s for API response to land
                for _ in range(6):
                    if lid in intercepted:
                        break
                    await asyncio.sleep(1)

                if lid in intercepted:
                    if intercepted[lid] is None:
                        print("API 404 → Removed")
                        results[lid] = None
                    else:
                        city = (intercepted[lid].get('location') or {}).get('city', '?')
                        print(f"OK — {city}")
                        results[lid] = intercepted[lid]
                else:
                    print("No API response captured")
                    results[lid] = 'no_data'

            except Exception as e:
                print(f"Error: {e}")
                results[lid] = 'no_data'

        await browser.close()

    return results


async def do_login():
    """Open browser for manual Properstar login; fetch favorites data from within the live session.

    Returns list of favorite listing dicts (raw API data), or None on failure.
    Uses page.evaluate(fetch) so auth state is the browser's own — no JWT export needed.
    """
    from playwright.async_api import async_playwright
    import json as _json

    intercepted_ids = []
    intercepted_details = {}

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=False)
        context = await browser.new_context()
        page = await context.new_page()

        # Intercept Properstar's OWN API responses — the browser adds the correct
        # Authorization header automatically. We just capture what the page fetches.
        async def on_response(response):
            if 'listglobally.com/api/v2/listings' not in response.url:
                return
            if response.status != 200:
                return
            try:
                body = await response.json()
            except Exception:
                return
            # Favorites list response
            if 'favorites' in response.url and 'listings' in body:
                ids = [l['id'] for l in body.get('listings', [])]
                if ids and not intercepted_ids:
                    intercepted_ids.extend(ids)
                    print(f"Intercepted {len(ids)} favorite IDs from page load")
            # Individual listing detail response
            import re as _re
            m = _re.search(r'/listings/(\d+)$', response.url)
            if m:
                lid = int(m.group(1))
                listing = body.get('listing', body)
                if listing.get('id'):
                    intercepted_details[lid] = listing

        page.on('response', on_response)

        print("Opening Properstar favorites page...")
        await page.goto("https://www.properstar.nl/favorites", timeout=120000)

        print("Log in if needed. Waiting for favorites to load (up to 5 min)...")
        for i in range(150):
            await asyncio.sleep(2)
            if intercepted_ids:
                print(f"Favorites loaded — {len(intercepted_ids)} properties found.")
                break
            if i % 15 == 0 and i > 0:
                print(f"  Waiting... ({i * 2}s). Not logged in? Log in now.")
        else:
            print("Timeout — no favorites API response intercepted within 5 minutes.")
            await browser.close()
            return None

        # Scroll/paginate to trigger loading of all favorites if page lazy-loads
        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        await asyncio.sleep(3)

        # For listings not loaded via detail responses, trigger them by scrolling
        missing = [lid for lid in intercepted_ids if lid not in intercepted_details]
        if missing:
            print(f"Waiting for {len(missing)} listing details to load...")
            await asyncio.sleep(5)
            missing = [lid for lid in intercepted_ids if lid not in intercepted_details]
            print(f"  {len(intercepted_details)}/{len(intercepted_ids)} detail responses captured")

        await browser.close()
        # Return whatever was intercepted — even IDs-only is useful for dedup
        return list(intercepted_details.values()) if intercepted_details else intercepted_ids

        await browser.close()
        return listings


def main():
    parser = argparse.ArgumentParser(description='Import Properstar favorites via API')
    parser.add_argument('--full', action='store_true', help='Re-import all (update existing)')
    parser.add_argument('--dry-run', action='store_true', help='Show what would be imported')
    parser.add_argument('--login', action='store_true', help='Force fresh login')
    parser.add_argument('--enrich-stubs', action='store_true', help='Visit stub listings in browser to check availability and capture data')
    args = parser.parse_args()

    # Enrich stubs: visit each stub listing page in Playwright
    if args.enrich_stubs:
        store = load()
        stubs = {
            int(re.search(r'/listing/(\d+)', url).group(1)): (url, prop)
            for url, prop in store.items()
            if 'properstar' in url
            and prop.get('status') != 'Removed'
            and not prop.get('title')
            and re.search(r'/listing/(\d+)', url)
        }
        if not stubs:
            print("No stubs found.")
            return
        print(f"Found {len(stubs)} stubs to enrich: {list(stubs.keys())}")
        results = asyncio.run(enrich_stubs(set(stubs.keys())))
        updated = removed = no_data = 0
        for lid, result in results.items():
            url, prop = stubs[lid]
            if result is None:
                upsert(store, url, {'status': 'Removed'})
                removed += 1
            elif result == 'no_data':
                no_data += 1
            else:
                fields = extract_fields(result)
                upsert(store, url, fields)
                updated += 1
        if updated + removed > 0:
            persist(store)
        print(f"\nStubs enriched: {updated} updated, {removed} marked Removed, {no_data} no API response")
        if updated > 0:
            print("Next: run `python3 pipeline.py --only geocode,score` to score the enriched stubs.")
        return

    # Proactive auth-age warning — surface before the 401 hits
    if TOKEN_PATH.exists() and not args.login:
        age_d = int((time.time() - TOKEN_PATH.stat().st_mtime) / 86400)
        if age_d > 30:
            print(f"⚠  Properstar token is {age_d}d old (typically expires ~30-60d).")
            print(f"   If next call 401s, re-run with --login.")

    # Login if needed — fetches all listings from within the live browser session
    if args.login or not TOKEN_PATH.exists():
        all_listings = asyncio.run(do_login())
        if all_listings is None:
            return

        store = load()
        existing_ps_ids = set()
        if not args.full:
            for url, prop in store.items():
                pid = prop.get('properstar_id')
                if pid:
                    existing_ps_ids.add(pid)
                elif 'properstar' in url:
                    m = re.search(r'/listing/(\d+)', url)
                    if m:
                        existing_ps_ids.add(int(m.group(1)))

        imported = errors = 0
        for detail in all_listings:
            # do_login() may return raw listing dicts or plain int IDs as fallback
            if isinstance(detail, int):
                url = f"https://www.properstar.nl/listing/{detail}"
                upsert(store, url, {'source': 'properstar', 'properstar_id': detail})
                imported += 1
                print(f"  Recorded ID {detail} (no detail yet — re-run --full later)")
                continue
            lid = detail.get('id')
            if not args.full and lid in existing_ps_ids:
                continue
            fields = extract_fields(detail)
            url = f"https://www.properstar.nl/listing/{lid}"
            try:
                upsert(store, url, fields)
                imported += 1
                city = fields.get('city', '?')
                price = fields.get('price')
                price_str = f"EUR {price:,}" if price else '?'
                print(f"  Imported {lid} — {city}, {price_str}")
            except Exception as e:
                errors += 1
                print(f"  FAIL {lid}: {e}")

        if imported > 0:
            persist(store)
        print(f"\nImported {imported} ({errors} errors)")
        return

    token = get_auth_token()
    if not token:
        print("No auth token found. Run with --login.")
        return

    session = make_session(token)

    # Test auth
    try:
        count_data = api_get(session, FAVORITES_URL, {'mode': 'NoListings', 'sort': '-LastActivityDate'})
        total = count_data.get('total', 0)
        print(f"Properstar favorites: {total}")
    except Exception as e:
        print(f"Auth failed: {e}")
        print("Run with --login to refresh.")
        return

    # Get all favorite IDs
    ids = fetch_favorite_ids(session)
    print(f"Fetched {len(ids)} favorite IDs")

    store = load()

    # Determine which need importing
    if args.full:
        to_import = ids
    else:
        existing_ps_ids = set()
        for url, prop in store.items():
            pid = prop.get('properstar_id')
            if pid:
                existing_ps_ids.add(pid)
            elif 'properstar' in url:
                m = re.search(r'/listing/(\d+)', url)
                if m:
                    existing_ps_ids.add(int(m.group(1)))

        to_import = [lid for lid in ids if lid not in existing_ps_ids]

    print(f"To import: {len(to_import)} {'(full refresh)' if args.full else 'new'}")

    # Import new/updated listings
    imported = 0
    errors = 0

    if args.dry_run:
        for lid in to_import:
            print(f"  Would import: {lid}")
    elif to_import:
        for i, lid in enumerate(to_import):
            print(f"  [{i+1}/{len(to_import)}] Listing {lid}...", end=' ', flush=True)

            try:
                detail = fetch_listing_detail(session, lid)
                fields = extract_fields(detail)

                url = f"https://www.properstar.nl/listing/{lid}"
                upsert(store, url, fields)
                imported += 1

                city = fields.get('city', '?')
                price = fields.get('price')
                price_str = f"EUR {price:,}" if price else '?'
                print(f"OK — {city}, {price_str}")

            except Exception as e:
                errors += 1
                print(f"FAIL ({e})")

            if imported > 0 and imported % 25 == 0:
                persist(store)

            time.sleep(0.1)

        print(f"\nImported {imported}/{len(to_import)} ({errors} errors)")
    else:
        print("Nothing new to import.")

    # Remove Properstar properties no longer in favorites
    api_id_set = set(ids)
    removed = 0
    for url, prop in list(store.items()):
        if 'properstar' not in url or prop.get('status') == 'Removed':
            continue
        pid = prop.get('properstar_id')
        if not pid:
            m = re.search(r'/listing/(\d+)', url)
            pid = int(m.group(1)) if m else None
        if pid and pid not in api_id_set:
            city = prop.get('city', '?')
            del store[url]
            print(f"  Deleted: {pid} ({city}) — no longer in favorites")
            removed += 1

    if removed:
        print(f"Deleted {removed} unfavorited properties")

    if imported > 0 or removed > 0:
        persist(store)
        print(f"Store: {len(store)} properties total")


if __name__ == '__main__':
    main()