NBA_Go/services/player_advanced_playoffs_scrape_service.go

// File: NBA_Go/services/player_advanced_scrape_service.go

package services

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/nprasad2077/NBA_Go/models"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

const (
	advancedURLFmt        = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
	advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
)

// urlForAdvSeason picks the correct URL based on isPlayoff.
func urlForAdvSeason(season int, isPlayoff bool) string {
	if isPlayoff {
		return fmt.Sprintf(advancedPlayoffURLFmt, season)
	}
	return fmt.Sprintf(advancedURLFmt, season)
}

// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs)
// and batch upserts the data into the PlayerAdvancedStat model.
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
	url := urlForAdvSeason(season, isPlayoff)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	htmlBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
	if err != nil {
		return err
	}

	// 1) Determine parent wrapper and table selector. This logic is complex because
	// the table is often hidden inside an HTML comment.
	var parentDivSelector, tableSelector string
	if isPlayoff {
		parentDivSelector = "#all_advanced_stats"
		tableSelector = "table#advanced_stats"
	} else {
		parentDivSelector = "#all_advanced"
		tableSelector = "table#advanced"
	}

	table := doc.Find(parentDivSelector + " " + tableSelector)
	if table.Length() == 0 {
		commentSel := doc.
			Find(parentDivSelector).
			Contents().
			FilterFunction(func(i int, s *goquery.Selection) bool {
				return goquery.NodeName(s) == "#comment"
			})

		if commentSel.Length() == 0 {
			return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season)
		}

		commentedHTML := commentSel.Nodes[0].FirstChild.Data
		innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
		if err != nil {
			return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err)
		}
		table = innerDoc.Find(tableSelector)
		if table.Length() == 0 {
			return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season)
		}
	}

	// 2) Collect the data-stat keys in header order.
	var headers []string
	table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
		if stat, ok := th.Attr("data-stat"); ok && stat != "" {
			headers = append(headers, stat)
		}
	})
	headers = append(headers, "player-additional")

	// --- BATCHING LOGIC START ---
	// Create a slice to hold all the player stats parsed from the page.
	var statsToUpsert []models.PlayerAdvancedStat

	// 3) Iterate each row and collect player stats into the slice.
	table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
		if tr.HasClass("thead") {
			return // skip repeated header rows
		}
		cells := tr.Find("th, td")
		data := make(map[string]string, len(headers))
		var playerID string

		cells.Each(func(i int, cell *goquery.Selection) {
			key := headers[i]
			data[key] = strings.TrimSpace(cell.Text())
			if id, ok := cell.Attr("data-append-csv"); ok {
				playerID = id
			}
		})
		if playerID == "" {
			return // not a real data row
		}
		data["player-additional"] = playerID

		// 4) Determine ExternalID, PlayerName, Team.
		extID := mustAtoi(data["rk"])
		if extID == 0 {
			extID = mustAtoi(data["ranker"])
		}

		playerName := data["player"]
		if playerName == "" {
			playerName = data["name_display"]
		}

		teamID := data["team_id"]
		if teamID == "" {
			teamID = data["team_name_abbr"]
		}

		// 5) Games column is always "g" in advanced tables.
		g := mustAtoi(data["games"])
		if g == 0 {
			g = mustAtoi(data["g"])
		}

		// 6) Map into your GORM model.
		stat := models.PlayerAdvancedStat{
			ExternalID:         extID,
			PlayerID:           playerID,
			PlayerName:         playerName,
			Position:           data["pos"],
			Age:                mustAtoi(data["age"]),
			Games:              g,
			MinutesPlayed:      mustAtoi(data["mp"]),
			PER:                mustParseFloat(data["per"]),
			TSPercent:          mustParseFloat(data["ts_pct"]),
			ThreePAR:           mustParseFloat(data["fg3a_per_fga_pct"]),
			FTR:                mustParseFloat(data["fta_per_fga_pct"]),
			OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
			DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
			TotalRBPercent:     mustParseFloat(data["trb_pct"]),
			AssistPercent:      mustParseFloat(data["ast_pct"]),
			StealPercent:       mustParseFloat(data["stl_pct"]),
			BlockPercent:       mustParseFloat(data["blk_pct"]),
			TurnoverPercent:    mustParseFloat(data["tov_pct"]),
			UsagePercent:       mustParseFloat(data["usg_pct"]),
			OffensiveWS:        mustParseFloat(data["ows"]),
			DefensiveWS:        mustParseFloat(data["dws"]),
			WinShares:          mustParseFloat(data["ws"]),
			WinSharesPer:       mustParseFloat(data["ws_per_48"]),
			OffensiveBox:       mustParseFloat(data["obpm"]),
			DefensiveBox:       mustParseFloat(data["dbpm"]),
			Box:                mustParseFloat(data["bpm"]),
			VORP:               mustParseFloat(data["vorp"]),
			Team:               teamID,
			Season:             season,
			IsPlayoff:          isPlayoff,
		}

		// Add the parsed stat object to our slice.
		statsToUpsert = append(statsToUpsert, stat)
	})

	// 7) Perform the batch upsert operation after collecting all rows.
	if len(statsToUpsert) > 0 {
		log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season)

		if err := db.Clauses(clause.OnConflict{
			Columns: []clause.Column{
				{Name: "player_id"},
				{Name: "season"},
				{Name: "team"},
				{Name: "is_playoff"},
			},
			DoUpdates: clause.AssignmentColumns([]string{
				"external_id", "player_name", "position", "age", "games",
				"minutes_played", "per", "ts_percent", "three_par", "ftr",
				"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
				"assist_percent", "steal_percent", "block_percent", "turnover_percent",
				"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
				"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
			}),
		}).Create(&statsToUpsert).Error; err != nil {
			log.Printf("Failed to batch upsert advanced player stats: %v", err)
			return err
		}
		log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season)
	} else {
		log.Printf("No advanced player data found to import for season %d.", season)
	}
	// --- BATCHING LOGIC END ---

	return nil
}