batch insert totals

import.go

@@ -9,33 +9,33 @@ import (
 	"github.com/nprasad2077/NBA_Go/utils"
 )
 
-// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
+// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
-func importPlayerAdvanced(db *gorm.DB) {
+// func importPlayerAdvanced(db *gorm.DB) {
-	for season := 2015; season <= 2019; season++ {
+// 	for season := 2013; season <= 2014; season++ {
-		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
+// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
-			log.Printf("advanced import failed for %d: %v", season, err)
+// 			log.Printf("advanced import failed for %d: %v", season, err)
-		}
+// 		}
-		log.Printf("Advanced import for season: %d", season)
+// 		log.Printf("Advanced import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+// 		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1000 * time.Millisecond)
+// 		utils.SleepWithJitter(1000 * time.Millisecond)
-	}
+// 	}
-}
+// }
 
-// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
+// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
-func importPlayerAdvancedPlayoffs(db *gorm.DB) {
+// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 2015; season <= 2019; season++ {
+// 	for season := 2013; season <= 2014; season++ {
-		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
+// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
-			log.Printf("advanced import failed for %d: %v", season, err)
+// 			log.Printf("advanced import failed for %d: %v", season, err)
-		}
+// 		}
-		log.Printf("Advanced Playoffs import for season: %d", season)
+// 		log.Printf("Advanced Playoffs import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+// 		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1250 * time.Millisecond)
+// 		utils.SleepWithJitter(1250 * time.Millisecond)
-	}
+// 	}
-}
+// }
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-    for season := 2015; season <= 2019; season++ {
+    for season := 2013; season <= 2014; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
             log.Printf("scraped totals import failed for %d: %v", season, err)
         }
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-    for season := 2015; season <= 2019; season++ {
+    for season := 2013; season <= 2014; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
             log.Printf("scraped playoffs import failed for %d: %v", season, err)
         }

main.go

@@ -45,11 +45,11 @@ func main() {
 		// Run all migrations + import steps exactly once
 		db := config.InitDB(true)
 
-		importPlayerAdvanced(db)
+		// importPlayerAdvanced(db)
-		log.Println("🎉 Player Advanced Import completed successfully")
+		// log.Println("🎉 Player Advanced Import completed successfully")
 
-		importPlayerAdvancedPlayoffs(db)
+		// importPlayerAdvancedPlayoffs(db)
-		log.Println("🎉 Player Advanced Playoffs Import completed successfully")
+		// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
 
 		importPlayerTotalsScrape(db)
 		log.Println("🎉 Player Totals (scraped) Import completed successfully")

services/player_total_scrape_service.go

@@ -30,7 +30,7 @@ func urlForSeason(season int, isPlayoff bool) string {
 }
 
 // FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs)
-// and upserts into PlayerTotalStat.
+// and batch upserts them into PlayerTotalStat for significantly better performance.
 func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
 	url := urlForSeason(season, isPlayoff)
 	req, err := http.NewRequest("GET", url, nil)
@@ -56,47 +56,26 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
 
 	table := doc.Find("table#totals_stats")
 	if table.Length() == 0 {
-        return fmt.Errorf("could not find table#totals_stats")
+		return fmt.Errorf("could not find table#totals_stats for season %d", season)
 	}
 
-    // table := doc.Find("table#totals_stats")
+	// 1) Collect the data-stat keys in header order.
-    // if table.Length() == 0 {
-    //     commentSel := doc.
-    //         Find("div#all_totals_stats").
-    //         Contents().
-    //         FilterFunction(func(i int, s *goquery.Selection) bool {
-    //             return goquery.NodeName(s) == "#comment"
-    //         })
-
-    //     if commentSel.Length() == 0 {
-    //         return fmt.Errorf("could not find table#totals_stats (even inside comment)")
-    //     }
-
-    //     commentedHTML := commentSel.Nodes[0].FirstChild.Data
-    //     innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
-    //     if err != nil {
-    //         return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err)
-    //     }
-    //     table = innerDoc.Find("table#totals_stats")
-    //     if table.Length() == 0 {
-    //         return fmt.Errorf("could not find table#totals_stats after un‐commenting")
-    //     }
-    // }
-
-    // 1) collect the data-stat keys in header order
 	var headers []string
 	table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
 		if stat, ok := th.Attr("data-stat"); ok && stat != "" {
 			headers = append(headers, stat)
 		}
 	})
-    // add the appended-player column
+	headers = append(headers, "player-additional") // Add the appended-player column.
-    headers = append(headers, "player-additional")
 
-    // 2) iterate rows
+	// --- BATCHING LOGIC START ---
+	// Create a slice to hold all the player stats parsed from the page.
+	var statsToUpsert []models.PlayerTotalStat
+
+	// 2) Iterate rows and collect all player stats into the slice.
 	table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
 		if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
-            return // skip header rows
+			return // Skip repeated header rows inside the table body.
 		}
 
 		cells := tr.Find("th, td")
@@ -111,36 +90,38 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
 				playerID = id
 			}
 		})
+
+		// Skip rows that aren't actual player data rows.
 		if playerID == "" {
 			return
 		}
 		data["player-additional"] = playerID
 
-    // 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”)
+		// 3a) Pick the right “ExternalID” key.
 		extID := mustAtoi(data["rk"])
 		if extID == 0 {
 			extID = mustAtoi(data["ranker"])
 		}
 
-    // 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”)
+		// 3b) Pick the right “PlayerName” key.
 		playerName := data["player"]
 		if playerName == "" {
 			playerName = data["name_display"]
 		}
 
-    // 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”)
+		// 3c) Pick the right “Team” key.
 		teamID := data["team_id"]
 		if teamID == "" {
 			teamID = data["team_name_abbr"]
 		}
 
-    // pick “games” → fallback to “g” if empty
+		// Pick “games”.
 		g := mustAtoi(data["games"])
 		if g == 0 {
 			g = mustAtoi(data["g"])
 		}
 
-    // pick “games_started” → fallback to “gs” if empty
+		// Pick “games_started”.
 		gs := mustAtoi(data["games_started"])
 		if gs == 0 {
 			gs = mustAtoi(data["gs"])
@@ -182,7 +163,15 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
 			IsPlayoff:       isPlayoff,
 		}
 
-        // 4) upsert on (player_id, season, team, is_playoff)
+		// Add the parsed stat object to our slice instead of writing to the DB immediately.
+		statsToUpsert = append(statsToUpsert, stat)
+	})
+
+	// 3) Perform the batch upsert operation after collecting all rows.
+	if len(statsToUpsert) > 0 {
+		log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season)
+
+		// GORM's OnConflict clause works with slices, performing the batch operation efficiently.
 		if err := db.Clauses(clause.OnConflict{
 			Columns: []clause.Column{
 				{Name: "player_id"},
@@ -202,10 +191,17 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
 				"assists", "steals", "blocks", "turnovers",
 				"personal_fouls", "points",
 			}),
-        }).Create(&stat).Error; err != nil {
+		}).Create(&statsToUpsert).Error; err != nil {
-            log.Printf("Failed upsert for %s: %v", stat.PlayerID, err)
+			// If the batch operation fails, log the error and return it.
+			log.Printf("Failed to batch upsert player total stats: %v", err)
+			return err
 		}
-    })
+
+		log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season)
+	} else {
+		log.Printf("No player data found to import for season %d.", season)
+	}
+	// --- BATCHING LOGIC END ---
 
 	return nil
 }