batch insert adv

import.go

@@ -9,33 +9,33 @@ import (
 	"github.com/nprasad2077/NBA_Go/utils"
 )
 
-// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
+// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
-// func importPlayerAdvanced(db *gorm.DB) {
+func importPlayerAdvanced(db *gorm.DB) {
-// 	for season := 2013; season <= 2014; season++ {
+	for season := 2000; season <= 2014; season++ {
-// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
+		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
-// 			log.Printf("advanced import failed for %d: %v", season, err)
+			log.Printf("advanced import failed for %d: %v", season, err)
-// 		}
+		}
-// 		log.Printf("Advanced import for season: %d", season)
+		log.Printf("Advanced import for season: %d", season)
-// 		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1100 * time.Millisecond)
-// 		utils.SleepWithJitter(1000 * time.Millisecond)
+		utils.SleepWithJitter(1000 * time.Millisecond)
-// 	}
+	}
-// }
+}
 
-// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
+// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
-// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
+func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-// 	for season := 2013; season <= 2014; season++ {
+	for season := 2000; season <= 2014; season++ {
-// 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
+		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
-// 			log.Printf("advanced import failed for %d: %v", season, err)
+			log.Printf("advanced import failed for %d: %v", season, err)
-// 		}
+		}
-// 		log.Printf("Advanced Playoffs import for season: %d", season)
+		log.Printf("Advanced Playoffs import for season: %d", season)
-// 		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1100 * time.Millisecond)
-// 		utils.SleepWithJitter(1250 * time.Millisecond)
+		utils.SleepWithJitter(1250 * time.Millisecond)
-// 	}
+	}
-// }
+}
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-    for season := 2013; season <= 2014; season++ {
+    for season := 2000; season <= 2012; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
             log.Printf("scraped totals import failed for %d: %v", season, err)
         }
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-    for season := 2013; season <= 2014; season++ {
+    for season := 2000; season <= 2012; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
             log.Printf("scraped playoffs import failed for %d: %v", season, err)
         }

main.go

@@ -45,11 +45,11 @@ func main() {
 		// Run all migrations + import steps exactly once
 		db := config.InitDB(true)
 
-		// importPlayerAdvanced(db)
+		importPlayerAdvanced(db)
-		// log.Println("🎉 Player Advanced Import completed successfully")
+		log.Println("🎉 Player Advanced Import completed successfully")
 
-		// importPlayerAdvancedPlayoffs(db)
+		importPlayerAdvancedPlayoffs(db)
-		// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
+		log.Println("🎉 Player Advanced Playoffs Import completed successfully")
 
 		importPlayerTotalsScrape(db)
 		log.Println("🎉 Player Totals (scraped) Import completed successfully")

services/player_advanced_playoffs_scrape_service.go

@@ -29,8 +29,8 @@ func urlForAdvSeason(season int, isPlayoff bool) string {
 	return fmt.Sprintf(advancedURLFmt, season)
 }
 
-// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table
+// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs)
-// (regular or playoffs) and upserts into the PlayerAdvancedStat model.
+// and batch upserts the data into the PlayerAdvancedStat model.
 func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
 	url := urlForAdvSeason(season, isPlayoff)
 	req, err := http.NewRequest("GET", url, nil)
@@ -54,9 +54,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 		return err
 	}
 
-    // 1) Determine parent wrapper and table selector
+	// 1) Determine parent wrapper and table selector. This logic is complex because
-    //    - Regular season: <div id="all_advanced"><!-- <table id="advanced">…</table> --></div>
+	// the table is often hidden inside an HTML comment.
-    //    - Playoffs:       <div id="all_advanced_stats"><!-- <table id="advanced_stats">…</table> --></div>
 	var parentDivSelector, tableSelector string
 	if isPlayoff {
 		parentDivSelector = "#all_advanced_stats"
@@ -66,10 +65,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 		tableSelector = "table#advanced"
 	}
 
-    // Attempt to find the table node directly (it will be inside a comment for regular season)
 	table := doc.Find(parentDivSelector + " " + tableSelector)
 	if table.Length() == 0 {
-        // Look for the commented‐out HTML inside parentDivSelector
 		commentSel := doc.
 			Find(parentDivSelector).
 			Contents().
@@ -78,32 +75,34 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 			})
 
 		if commentSel.Length() == 0 {
-            return fmt.Errorf("could not find advanced stats table (even inside comment)")
+			return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season)
 		}
 
-        // Extract the raw HTML string from the comment, then re‐parse
 		commentedHTML := commentSel.Nodes[0].FirstChild.Data
 		innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
 		if err != nil {
-            return fmt.Errorf("failed to parse commented advanced HTML: %w", err)
+			return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err)
 		}
 		table = innerDoc.Find(tableSelector)
 		if table.Length() == 0 {
-            return fmt.Errorf("could not find advanced stats table after un‐commenting")
+			return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season)
 		}
 	}
 
-    // 2) Collect the data-stat keys in header order
+	// 2) Collect the data-stat keys in header order.
 	var headers []string
 	table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
 		if stat, ok := th.Attr("data-stat"); ok && stat != "" {
 			headers = append(headers, stat)
 		}
 	})
-    // Add our appended‐player column
 	headers = append(headers, "player-additional")
 
-    // 3) Iterate each row
+	// --- BATCHING LOGIC START ---
+	// Create a slice to hold all the player stats parsed from the page.
+	var statsToUpsert []models.PlayerAdvancedStat
+
+	// 3) Iterate each row and collect player stats into the slice.
 	table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
 		if tr.HasClass("thead") {
 			return // skip repeated header rows
@@ -120,14 +119,11 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 			}
 		})
 		if playerID == "" {
-            // not a real data row
+			return // not a real data row
-            return
 		}
 		data["player-additional"] = playerID
 
-        // 4) Determine ExternalID, PlayerName, Team
+		// 4) Determine ExternalID, PlayerName, Team.
-        //    • Playoff pages use "rk", "player", "team_id"
-        //    • Regular‐season advanced uses "ranker", "name_display", "team_name_abbr"
 		extID := mustAtoi(data["rk"])
 		if extID == 0 {
 			extID = mustAtoi(data["ranker"])
@@ -143,13 +139,13 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 			teamID = data["team_name_abbr"]
 		}
 
-        // 5) Games column is always "g" in advanced (playoffs or season)
+		// 5) Games column is always "g" in advanced tables.
 		g := mustAtoi(data["games"])
 		if g == 0 {
 			g = mustAtoi(data["g"])
 		}
 
-        // 6) Map into your GORM model
+		// 6) Map into your GORM model.
 		stat := models.PlayerAdvancedStat{
 			ExternalID:         extID,
 			PlayerID:           playerID,
@@ -183,7 +179,14 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 			IsPlayoff:          isPlayoff,
 		}
 
-        // 7) Upsert on (player_id, season, team, is_playoff)
+		// Add the parsed stat object to our slice.
+		statsToUpsert = append(statsToUpsert, stat)
+	})
+
+	// 7) Perform the batch upsert operation after collecting all rows.
+	if len(statsToUpsert) > 0 {
+		log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season)
+
 		if err := db.Clauses(clause.OnConflict{
 			Columns: []clause.Column{
 				{Name: "player_id"},
@@ -193,17 +196,21 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
 			},
 			DoUpdates: clause.AssignmentColumns([]string{
 				"external_id", "player_name", "position", "age", "games",
-                "minutes_played",
+				"minutes_played", "per", "ts_percent", "three_par", "ftr",
-                "per", "ts_percent", "three_par", "ftr",           // ← use three_par & ftr
 				"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
 				"assist_percent", "steal_percent", "block_percent", "turnover_percent",
 				"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
 				"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
 			}),
-        }).Create(&stat).Error; err != nil {
+		}).Create(&statsToUpsert).Error; err != nil {
-            log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
+			log.Printf("Failed to batch upsert advanced player stats: %v", err)
+			return err
 		}
-    })
+		log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season)
+	} else {
+		log.Printf("No advanced player data found to import for season %d.", season)
+	}
+	// --- BATCHING LOGIC END ---
 
 	return nil
 }