NBA_Go/services/player_total_scrape_service.go

// File: NBA_Go/services/player_total_scrape_service.go

package services

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/nprasad2077/NBA_Go/models"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

const (
	regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html"
	playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html"
)

// urlForSeason chooses regular vs. playoff URL.
func urlForSeason(season int, isPlayoff bool) string {
	if isPlayoff {
		return fmt.Sprintf(playoffURLFmt, season)
	}
	return fmt.Sprintf(regularURLFmt, season)
}

// FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs)
// and batch upserts them into PlayerTotalStat for significantly better performance.
func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
	url := urlForSeason(season, isPlayoff)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
	if err != nil {
		return err
	}

	table := doc.Find("table#totals_stats")
	if table.Length() == 0 {
		return fmt.Errorf("could not find table#totals_stats for season %d", season)
	}

	// 1) Collect the data-stat keys in header order.
	var headers []string
	table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
		if stat, ok := th.Attr("data-stat"); ok && stat != "" {
			headers = append(headers, stat)
		}
	})
	headers = append(headers, "player-additional") // Add the appended-player column.

	// --- BATCHING LOGIC START ---
	// Create a slice to hold all the player stats parsed from the page.
	var statsToUpsert []models.PlayerTotalStat

	// 2) Iterate rows and collect all player stats into the slice.
	table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
		if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
			return // Skip repeated header rows inside the table body.
		}

		cells := tr.Find("th, td")
		data := make(map[string]string, len(headers))
		var playerID string

		cells.Each(func(i int, cell *goquery.Selection) {
			text := strings.TrimSpace(cell.Text())
			key := headers[i]
			data[key] = text
			if id, ok := cell.Attr("data-append-csv"); ok {
				playerID = id
			}
		})

		// Skip rows that aren't actual player data rows.
		if playerID == "" {
			return
		}
		data["player-additional"] = playerID

		// 3a) Pick the right “ExternalID” key.
		extID := mustAtoi(data["rk"])
		if extID == 0 {
			extID = mustAtoi(data["ranker"])
		}

		// 3b) Pick the right “PlayerName” key.
		playerName := data["player"]
		if playerName == "" {
			playerName = data["name_display"]
		}

		// 3c) Pick the right “Team” key.
		teamID := data["team_id"]
		if teamID == "" {
			teamID = data["team_name_abbr"]
		}

		// Pick “games”.
		g := mustAtoi(data["games"])
		if g == 0 {
			g = mustAtoi(data["g"])
		}

		// Pick “games_started”.
		gs := mustAtoi(data["games_started"])
		if gs == 0 {
			gs = mustAtoi(data["gs"])
		}

		stat := models.PlayerTotalStat{
			ExternalID:      extID,
			PlayerID:        playerID,
			PlayerName:      playerName,
			Position:        data["pos"],
			Age:             mustAtoi(data["age"]),
			Games:           g,
			GamesStarted:    gs,
			MinutesPG:       mustParseFloat(data["mp"]),
			FieldGoals:      mustAtoi(data["fg"]),
			FieldAttempts:   mustAtoi(data["fga"]),
			FieldPercent:    mustParseFloat(data["fg_pct"]),
			ThreeFG:         mustAtoi(data["fg3"]),
			ThreeAttempts:   mustAtoi(data["fg3a"]),
			ThreePercent:    mustParseFloat(data["fg3_pct"]),
			TwoFG:           mustAtoi(data["fg2"]),
			TwoAttempts:     mustAtoi(data["fg2a"]),
			TwoPercent:      mustParseFloat(data["fg2_pct"]),
			EffectFGPercent: mustParseFloat(data["efg_pct"]),
			FT:              mustAtoi(data["ft"]),
			FTAttempts:      mustAtoi(data["fta"]),
			FTPercent:       mustParseFloat(data["ft_pct"]),
			OffensiveRB:     mustAtoi(data["orb"]),
			DefensiveRB:     mustAtoi(data["drb"]),
			TotalRB:         mustAtoi(data["trb"]),
			Assists:         mustAtoi(data["ast"]),
			Steals:          mustAtoi(data["stl"]),
			Blocks:          mustAtoi(data["blk"]),
			Turnovers:       mustAtoi(data["tov"]),
			PersonalFouls:   mustAtoi(data["pf"]),
			Points:          mustAtoi(data["pts"]),
			Team:            teamID,
			Season:          season,
			IsPlayoff:       isPlayoff,
		}

		// Add the parsed stat object to our slice instead of writing to the DB immediately.
		statsToUpsert = append(statsToUpsert, stat)
	})

	// 3) Perform the batch upsert operation after collecting all rows.
	if len(statsToUpsert) > 0 {
		log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season)

		// GORM's OnConflict clause works with slices, performing the batch operation efficiently.
		if err := db.Clauses(clause.OnConflict{
			Columns: []clause.Column{
				{Name: "player_id"},
				{Name: "season"},
				{Name: "team"},
				{Name: "is_playoff"},
			},
			DoUpdates: clause.AssignmentColumns([]string{
				"external_id", "player_name", "position", "age",
				"games", "games_started", "minutes_pg",
				"field_goals", "field_attempts", "field_percent",
				"three_fg", "three_attempts", "three_percent",
				"two_fg", "two_attempts", "two_percent",
				"effect_fg_percent",
				"ft", "ft_attempts", "ft_percent",
				"offensive_rb", "defensive_rb", "total_rb",
				"assists", "steals", "blocks", "turnovers",
				"personal_fouls", "points",
			}),
		}).Create(&statsToUpsert).Error; err != nil {
			// If the batch operation fails, log the error and return it.
			log.Printf("Failed to batch upsert player total stats: %v", err)
			return err
		}

		log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season)
	} else {
		log.Printf("No player data found to import for season %d.", season)
	}
	// --- BATCHING LOGIC END ---

	return nil
}