batch insert totals

import.go

@@ -9,33 +9,33 @@ import (
 	"github.com/nprasad2077/NBA_Go/utils"
 )
 
-// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
+// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
-func importPlayerAdvanced(db *gorm.DB) {
+// func importPlayerAdvanced(db *gorm.DB) {
-	for season := 2015; season <= 2019; season++ {
+// 	for season := 2013; season <= 2014; season++ {
-		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
+// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
-			log.Printf("advanced import failed for %d: %v", season, err)
+// 			log.Printf("advanced import failed for %d: %v", season, err)
-		}
+// 		}
-		log.Printf("Advanced import for season: %d", season)
+// 		log.Printf("Advanced import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+// 		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1000 * time.Millisecond)
+// 		utils.SleepWithJitter(1000 * time.Millisecond)
-	}
+// 	}
-}
+// }
 
-// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
+// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
-func importPlayerAdvancedPlayoffs(db *gorm.DB) {
+// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 2015; season <= 2019; season++ {
+// 	for season := 2013; season <= 2014; season++ {
-		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
+// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
-			log.Printf("advanced import failed for %d: %v", season, err)
+// 			log.Printf("advanced import failed for %d: %v", season, err)
-		}
+// 		}
-		log.Printf("Advanced Playoffs import for season: %d", season)
+// 		log.Printf("Advanced Playoffs import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+// 		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1250 * time.Millisecond)
+// 		utils.SleepWithJitter(1250 * time.Millisecond)
-	}
+// 	}
-}
+// }
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-    for season := 2015; season <= 2019; season++ {
+    for season := 2013; season <= 2014; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
             log.Printf("scraped totals import failed for %d: %v", season, err)
         }
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-    for season := 2015; season <= 2019; season++ {
+    for season := 2013; season <= 2014; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
             log.Printf("scraped playoffs import failed for %d: %v", season, err)
         }

main.go

@@ -45,11 +45,11 @@ func main() {
 		// Run all migrations + import steps exactly once
 		db := config.InitDB(true)
 
-		importPlayerAdvanced(db)
+		// importPlayerAdvanced(db)
-		log.Println("🎉 Player Advanced Import completed successfully")
+		// log.Println("🎉 Player Advanced Import completed successfully")
 
-		importPlayerAdvancedPlayoffs(db)
+		// importPlayerAdvancedPlayoffs(db)
-		log.Println("🎉 Player Advanced Playoffs Import completed successfully")
+		// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
 
 		importPlayerTotalsScrape(db)
 		log.Println("🎉 Player Totals (scraped) Import completed successfully")

services/player_total_scrape_service.go

@@ -3,209 +3,205 @@
 package services
 
 import (
-    "bytes"
+	"bytes"
-    "fmt"
+	"fmt"
-    "io"
+	"io"
-    "log"
+	"log"
-    "net/http"
+	"net/http"
-    "strings"
+	"strings"
 
-    "github.com/PuerkitoBio/goquery"
+	"github.com/PuerkitoBio/goquery"
-    "github.com/nprasad2077/NBA_Go/models"
+	"github.com/nprasad2077/NBA_Go/models"
-    "gorm.io/gorm"
+	"gorm.io/gorm"
-    "gorm.io/gorm/clause"
+	"gorm.io/gorm/clause"
 )
 
 const (
-    regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html"
+	regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html"
-    playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html"
+	playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html"
 )
 
 // urlForSeason chooses regular vs. playoff URL.
 func urlForSeason(season int, isPlayoff bool) string {
-    if isPlayoff {
+	if isPlayoff {
-        return fmt.Sprintf(playoffURLFmt, season)
+		return fmt.Sprintf(playoffURLFmt, season)
-    }
+	}
-    return fmt.Sprintf(regularURLFmt, season)
+	return fmt.Sprintf(regularURLFmt, season)
 }
 
 // FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs)
-// and upserts into PlayerTotalStat.
+// and batch upserts them into PlayerTotalStat for significantly better performance.
 func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
-    url := urlForSeason(season, isPlayoff)
+	url := urlForSeason(season, isPlayoff)
-    req, err := http.NewRequest("GET", url, nil)
+	req, err := http.NewRequest("GET", url, nil)
-    if err != nil {
+	if err != nil {
-        return err
+		return err
-    }
+	}
-    req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
-    resp, err := http.DefaultClient.Do(req)
+	resp, err := http.DefaultClient.Do(req)
-    if err != nil {
+	if err != nil {
-        return err
+		return err
-    }
+	}
-    defer resp.Body.Close()
+	defer resp.Body.Close()
 
-    body, err := io.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
-    if err != nil {
+	if err != nil {
-        return err 
+		return err
-    }
+	}
 
-    doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
-    if err != nil {
+	if err != nil {
-        return err
+		return err
-    }
+	}
 
-    table := doc.Find("table#totals_stats")
+	table := doc.Find("table#totals_stats")
-    if table.Length() == 0 {
+	if table.Length() == 0 {
-        return fmt.Errorf("could not find table#totals_stats")
+		return fmt.Errorf("could not find table#totals_stats for season %d", season)
-    }
+	}
 
-    // table := doc.Find("table#totals_stats")
+	// 1) Collect the data-stat keys in header order.
-    // if table.Length() == 0 {
+	var headers []string
-    //     commentSel := doc.
+	table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
-    //         Find("div#all_totals_stats").
+		if stat, ok := th.Attr("data-stat"); ok && stat != "" {
-    //         Contents().
+			headers = append(headers, stat)
-    //         FilterFunction(func(i int, s *goquery.Selection) bool {
+		}
-    //             return goquery.NodeName(s) == "#comment"
+	})
-    //         })
+	headers = append(headers, "player-additional") // Add the appended-player column.
 
-    //     if commentSel.Length() == 0 {
+	// --- BATCHING LOGIC START ---
-    //         return fmt.Errorf("could not find table#totals_stats (even inside comment)")
+	// Create a slice to hold all the player stats parsed from the page.
-    //     }
+	var statsToUpsert []models.PlayerTotalStat
 
-    //     commentedHTML := commentSel.Nodes[0].FirstChild.Data
+	// 2) Iterate rows and collect all player stats into the slice.
-    //     innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
+	table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
-    //     if err != nil {
+		if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
-    //         return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err)
+			return // Skip repeated header rows inside the table body.
-    //     }
+		}
-    //     table = innerDoc.Find("table#totals_stats")
-    //     if table.Length() == 0 {
-    //         return fmt.Errorf("could not find table#totals_stats after un‐commenting")
-    //     }
-    // }
 
-    // 1) collect the data-stat keys in header order
+		cells := tr.Find("th, td")
-    var headers []string
+		data := make(map[string]string, len(headers))
-    table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
+		var playerID string
-        if stat, ok := th.Attr("data-stat"); ok && stat != "" {
-            headers = append(headers, stat)
-        }
-    })
-    // add the appended-player column
-    headers = append(headers, "player-additional")
 
-    // 2) iterate rows
+		cells.Each(func(i int, cell *goquery.Selection) {
-    table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
+			text := strings.TrimSpace(cell.Text())
-        if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
+			key := headers[i]
-            return // skip header rows
+			data[key] = text
-        }
+			if id, ok := cell.Attr("data-append-csv"); ok {
+				playerID = id
+			}
+		})
 
-        cells := tr.Find("th, td")
+		// Skip rows that aren't actual player data rows.
-        data := make(map[string]string, len(headers))
+		if playerID == "" {
-        var playerID string
+			return
+		}
+		data["player-additional"] = playerID
 
-        cells.Each(func(i int, cell *goquery.Selection) {
+		// 3a) Pick the right “ExternalID” key.
-            text := strings.TrimSpace(cell.Text())
+		extID := mustAtoi(data["rk"])
-            key := headers[i]
+		if extID == 0 {
-            data[key] = text
+			extID = mustAtoi(data["ranker"])
-            if id, ok := cell.Attr("data-append-csv"); ok {
+		}
-                playerID = id
-            }
-        })
-        if playerID == "" {
-            return
-        }
-        data["player-additional"] = playerID
 
-    // 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”)
+		// 3b) Pick the right “PlayerName” key.
-    extID := mustAtoi(data["rk"])
+		playerName := data["player"]
-    if extID == 0 {
+		if playerName == "" {
-        extID = mustAtoi(data["ranker"])
+			playerName = data["name_display"]
-    }
+		}
 
-    // 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”)
+		// 3c) Pick the right “Team” key.
-    playerName := data["player"]
+		teamID := data["team_id"]
-    if playerName == "" {
+		if teamID == "" {
-        playerName = data["name_display"]
+			teamID = data["team_name_abbr"]
-    }
+		}
 
-    // 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”)
+		// Pick “games”.
-    teamID := data["team_id"]
+		g := mustAtoi(data["games"])
-    if teamID == "" {
+		if g == 0 {
-        teamID = data["team_name_abbr"]
+			g = mustAtoi(data["g"])
-    }
+		}
 
-    // pick “games” → fallback to “g” if empty
+		// Pick “games_started”.
-    g := mustAtoi(data["games"])
+		gs := mustAtoi(data["games_started"])
-    if g == 0 {
+		if gs == 0 {
-        g = mustAtoi(data["g"])
+			gs = mustAtoi(data["gs"])
-    }
+		}
 
-    // pick “games_started” → fallback to “gs” if empty
+		stat := models.PlayerTotalStat{
-    gs := mustAtoi(data["games_started"])
+			ExternalID:      extID,
-    if gs == 0 {
+			PlayerID:        playerID,
-        gs = mustAtoi(data["gs"])
+			PlayerName:      playerName,
-    }
+			Position:        data["pos"],
+			Age:             mustAtoi(data["age"]),
+			Games:           g,
+			GamesStarted:    gs,
+			MinutesPG:       mustParseFloat(data["mp"]),
+			FieldGoals:      mustAtoi(data["fg"]),
+			FieldAttempts:   mustAtoi(data["fga"]),
+			FieldPercent:    mustParseFloat(data["fg_pct"]),
+			ThreeFG:         mustAtoi(data["fg3"]),
+			ThreeAttempts:   mustAtoi(data["fg3a"]),
+			ThreePercent:    mustParseFloat(data["fg3_pct"]),
+			TwoFG:           mustAtoi(data["fg2"]),
+			TwoAttempts:     mustAtoi(data["fg2a"]),
+			TwoPercent:      mustParseFloat(data["fg2_pct"]),
+			EffectFGPercent: mustParseFloat(data["efg_pct"]),
+			FT:              mustAtoi(data["ft"]),
+			FTAttempts:      mustAtoi(data["fta"]),
+			FTPercent:       mustParseFloat(data["ft_pct"]),
+			OffensiveRB:     mustAtoi(data["orb"]),
+			DefensiveRB:     mustAtoi(data["drb"]),
+			TotalRB:         mustAtoi(data["trb"]),
+			Assists:         mustAtoi(data["ast"]),
+			Steals:          mustAtoi(data["stl"]),
+			Blocks:          mustAtoi(data["blk"]),
+			Turnovers:       mustAtoi(data["tov"]),
+			PersonalFouls:   mustAtoi(data["pf"]),
+			Points:          mustAtoi(data["pts"]),
+			Team:            teamID,
+			Season:          season,
+			IsPlayoff:       isPlayoff,
+		}
 
-    stat := models.PlayerTotalStat{
+		// Add the parsed stat object to our slice instead of writing to the DB immediately.
-        ExternalID:    extID,
+		statsToUpsert = append(statsToUpsert, stat)
-        PlayerID:      playerID,
+	})
-        PlayerName:    playerName,
-        Position:      data["pos"],
-        Age:           mustAtoi(data["age"]),
-        Games:         g,
-        GamesStarted:  gs,
-        MinutesPG:     mustParseFloat(data["mp"]),
-        FieldGoals:    mustAtoi(data["fg"]),
-        FieldAttempts: mustAtoi(data["fga"]),
-        FieldPercent:  mustParseFloat(data["fg_pct"]),
-        ThreeFG:       mustAtoi(data["fg3"]),
-        ThreeAttempts: mustAtoi(data["fg3a"]),
-        ThreePercent:  mustParseFloat(data["fg3_pct"]),
-        TwoFG:         mustAtoi(data["fg2"]),
-        TwoAttempts:   mustAtoi(data["fg2a"]),
-        TwoPercent:    mustParseFloat(data["fg2_pct"]),
-        EffectFGPercent: mustParseFloat(data["efg_pct"]),
-        FT:            mustAtoi(data["ft"]),
-        FTAttempts:    mustAtoi(data["fta"]),
-        FTPercent:     mustParseFloat(data["ft_pct"]),
-        OffensiveRB:   mustAtoi(data["orb"]),
-        DefensiveRB:   mustAtoi(data["drb"]),
-        TotalRB:       mustAtoi(data["trb"]),
-        Assists:       mustAtoi(data["ast"]),
-        Steals:        mustAtoi(data["stl"]),
-        Blocks:        mustAtoi(data["blk"]),
-        Turnovers:     mustAtoi(data["tov"]),
-        PersonalFouls: mustAtoi(data["pf"]),
-        Points:        mustAtoi(data["pts"]),
-        Team:          teamID,
-        Season:        season,
-        IsPlayoff:     isPlayoff,
-    }
 
-        // 4) upsert on (player_id, season, team, is_playoff)
+	// 3) Perform the batch upsert operation after collecting all rows.
-        if err := db.Clauses(clause.OnConflict{
+	if len(statsToUpsert) > 0 {
-            Columns: []clause.Column{
+		log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season)
-                {Name: "player_id"},
-                {Name: "season"},
-                {Name: "team"},
-                {Name: "is_playoff"},
-            },
-            DoUpdates: clause.AssignmentColumns([]string{
-                "external_id", "player_name", "position", "age",
-                "games", "games_started", "minutes_pg",
-                "field_goals", "field_attempts", "field_percent",
-                "three_fg", "three_attempts", "three_percent",
-                "two_fg", "two_attempts", "two_percent",
-                "effect_fg_percent",
-                "ft", "ft_attempts", "ft_percent",
-                "offensive_rb", "defensive_rb", "total_rb",
-                "assists", "steals", "blocks", "turnovers",
-                "personal_fouls", "points",
-            }),
-        }).Create(&stat).Error; err != nil {
-            log.Printf("Failed upsert for %s: %v", stat.PlayerID, err)
-        }
-    })
 
-    return nil
+		// GORM's OnConflict clause works with slices, performing the batch operation efficiently.
+		if err := db.Clauses(clause.OnConflict{
+			Columns: []clause.Column{
+				{Name: "player_id"},
+				{Name: "season"},
+				{Name: "team"},
+				{Name: "is_playoff"},
+			},
+			DoUpdates: clause.AssignmentColumns([]string{
+				"external_id", "player_name", "position", "age",
+				"games", "games_started", "minutes_pg",
+				"field_goals", "field_attempts", "field_percent",
+				"three_fg", "three_attempts", "three_percent",
+				"two_fg", "two_attempts", "two_percent",
+				"effect_fg_percent",
+				"ft", "ft_attempts", "ft_percent",
+				"offensive_rb", "defensive_rb", "total_rb",
+				"assists", "steals", "blocks", "turnovers",
+				"personal_fouls", "points",
+			}),
+		}).Create(&statsToUpsert).Error; err != nil {
+			// If the batch operation fails, log the error and return it.
+			log.Printf("Failed to batch upsert player total stats: %v", err)
+			return err
+		}
+
+		log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season)
+	} else {
+		log.Printf("No player data found to import for season %d.", season)
+	}
+	// --- BATCHING LOGIC END ---
+
+	return nil
 }