scraper.py

"""Main scraper class for Football Ticket Net."""

import re
import time
from typing import List, Optional

import requests
from bs4 import BeautifulSoup

from config import (
    BASE_URL,
    SCRAPINGANT_API_URL,
    TOURNAMENTS,
    REQUEST_TIMEOUT,
    BROWSER_RENDERING,
    ITEMS_PER_PAGE,
)
from models import Match, MatchCollection
from utils import (
    normalize_url,
    clean_text,
    parse_match_title,
    parse_location,
    build_tournament_url,
    format_date,
    format_time,
)


class FootballTicketNetScraper:
    """Scraper for Football Ticket Net match listings."""

    def __init__(self, api_key: str, verbose: bool = False):
        """
        Initialize the scraper.

        Args:
            api_key: ScrapingAnt API key
            verbose: Enable verbose output
        """
        self.api_key = api_key
        self.verbose = verbose
        self.collection = MatchCollection()

    def _log(self, message: str) -> None:
        """Print message if verbose mode is enabled."""
        if self.verbose:
            print(message)

    def _fetch_page(self, url: str) -> Optional[str]:
        """
        Fetch a page using ScrapingAnt API.

        Args:
            url: URL to fetch

        Returns:
            HTML content or None if failed
        """
        params = {
            "url": url,
            "x-api-key": self.api_key,
            "browser": str(BROWSER_RENDERING).lower(),
            "proxy_country": "GB",
        }

        self._log(f"Fetching: {url}")

        try:
            response = requests.get(
                SCRAPINGANT_API_URL,
                params=params,
                timeout=REQUEST_TIMEOUT,
            )
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            self._log(f"Error fetching {url}: {e}")
            return None

    def _parse_match_cards(
        self, html: str, tournament: str = ""
    ) -> List[Match]:
        """
        Parse match cards from HTML content.

        Args:
            html: HTML content
            tournament: Tournament name for context

        Returns:
            List of parsed Match objects
        """
        soup = BeautifulSoup(html, "lxml")
        matches = []

        # Find all event cards
        event_cards = soup.select(".events_box, .ticktet_sec.team-event")

        self._log(f"Found {len(event_cards)} match cards")

        for card in event_cards:
            try:
                match_data = self._extract_match_data(card, tournament)
                if match_data:
                    match = Match.from_parsed_data(match_data)
                    matches.append(match)
            except Exception as e:
                self._log(f"Error parsing match card: {e}")
                continue

        return matches

    def _extract_match_data(
        self, card, tournament: str = ""
    ) -> Optional[dict]:
        """
        Extract match data from a card element.

        Args:
            card: BeautifulSoup element for the match card
            tournament: Tournament name for context

        Returns:
            Dictionary of match data or None
        """
        # Get event ID from data attribute
        event_id = card.get("data-event", "")

        # Get title and URL
        title_link = card.select_one(".team_info h3 a")
        if not title_link:
            return None

        match_title = clean_text(title_link.get_text())
        event_url = normalize_url(title_link.get("href", ""))

        # Parse home and away teams from title
        home_team, away_team = parse_match_title(match_title)

        # Get competition name
        competition_elem = card.select_one(".team_info > span")
        competition = ""
        if competition_elem:
            competition = clean_text(competition_elem.get_text())

        # Get venue
        venue_elem = card.select_one(".team_info .place i")
        venue = ""
        if venue_elem:
            venue = clean_text(venue_elem.get_text())

        # Get location (city, country)
        location_elem = card.select_one(".team_info .place .desktop-only")
        city, country = "", ""
        if location_elem:
            location_text = location_elem.get_text()
            city, country = parse_location(location_text)

        # Get date and time
        date_elem = card.select_one(".team_date span:first-child")
        time_elem = card.select_one(".team_date span:nth-child(2)")

        date = ""
        if date_elem:
            date = format_date(date_elem.get_text())

        match_time = ""
        if time_elem:
            match_time = format_time(time_elem.get_text())

        # Get team logos
        logo_imgs = card.select(".team_vs img")
        home_logo_url = None
        away_logo_url = None

        if len(logo_imgs) >= 1:
            home_logo_url = normalize_url(logo_imgs[0].get("src", ""))
        if len(logo_imgs) >= 2:
            away_logo_url = normalize_url(logo_imgs[1].get("src", ""))

        # Use event ID from URL if not found in data attribute
        if not event_id:
            event_id = event_url.rstrip("/").split("/")[-1]

        return {
            "event_id": event_id,
            "home_team": home_team,
            "away_team": away_team,
            "match_title": match_title,
            "competition": competition or tournament,
            "venue": venue,
            "city": city,
            "country": country,
            "date": date,
            "time": match_time,
            "event_url": event_url,
            "home_logo_url": home_logo_url,
            "away_logo_url": away_logo_url,
        }

    def _get_max_page(self, html: str) -> int:
        """
        Get the maximum page number from pagination.

        Args:
            html: HTML content

        Returns:
            Maximum page number
        """
        soup = BeautifulSoup(html, "lxml")

        # Find pagination links
        page_links = soup.select("a[href*='page=']")

        max_page = 1
        for link in page_links:
            href = link.get("href", "")
            match = re.search(r"page=(\d+)", href)
            if match:
                page_num = int(match.group(1))
                max_page = max(max_page, page_num)

        return max_page

    def scrape_tournament(
        self,
        tournament_key: str,
        max_pages: Optional[int] = None,
    ) -> int:
        """
        Scrape matches from a tournament.

        Args:
            tournament_key: Tournament key (e.g., 'premier-league')
            max_pages: Maximum pages to scrape (None for all)

        Returns:
            Number of matches found
        """
        if tournament_key not in TOURNAMENTS:
            self._log(f"Unknown tournament: {tournament_key}")
            return 0

        tournament_path = TOURNAMENTS[tournament_key]

        # Fetch first page to get pagination info
        url = build_tournament_url(tournament_path, 1)
        html = self._fetch_page(url)

        if not html:
            return 0

        # Get max pages
        total_pages = self._get_max_page(html)
        if max_pages:
            total_pages = min(total_pages, max_pages)

        self._log(f"Scraping {total_pages} pages from {tournament_key}")

        # Parse first page
        matches = self._parse_match_cards(html, tournament_key)
        added = self.collection.add_many(matches)

        self._log(
            f"Page 1: Added {added} matches (total: {len(self.collection)})"
        )

        # Scrape remaining pages
        for page in range(2, total_pages + 1):
            time.sleep(1)  # Rate limiting

            url = build_tournament_url(tournament_path, page)
            html = self._fetch_page(url)

            if not html:
                continue

            matches = self._parse_match_cards(html, tournament_key)
            added = self.collection.add_many(matches)

            self._log(
                f"Page {page}: Added {added} matches "
                f"(total: {len(self.collection)})"
            )

        return len(self.collection)

    def scrape_multiple_tournaments(
        self,
        tournament_keys: Optional[List[str]] = None,
        max_pages_per_tournament: Optional[int] = None,
    ) -> int:
        """
        Scrape matches from multiple tournaments.

        Args:
            tournament_keys: List of tournament keys, or None for all
            max_pages_per_tournament: Maximum pages per tournament

        Returns:
            Total number of matches found
        """
        if tournament_keys is None:
            tournament_keys = list(TOURNAMENTS.keys())

        total = 0

        for tournament in tournament_keys:
            count = self.scrape_tournament(
                tournament, max_pages=max_pages_per_tournament
            )
            total += count

            # Delay between tournaments
            if tournament != tournament_keys[-1]:
                time.sleep(1)

        return total

    def scrape_url(
        self, url: str, tournament: str = ""
    ) -> int:
        """
        Scrape matches from a specific URL.

        Args:
            url: Full URL to scrape
            tournament: Tournament name for context

        Returns:
            Number of matches found
        """
        html = self._fetch_page(url)

        if not html:
            return 0

        matches = self._parse_match_cards(html, tournament)
        added = self.collection.add_many(matches)

        self._log(
            f"Added {added} matches from URL (total: {len(self.collection)})"
        )
        return added

    def get_collection(self) -> MatchCollection:
        """Get the match collection."""
        return self.collection

    def clear(self) -> None:
        """Clear the match collection."""
        self.collection = MatchCollection()