box score scrape service updates batch handle

Dockerfile

@@ -4,8 +4,8 @@ FROM golang:1.24-bullseye AS builder
 ENV CGO_ENABLED=1
 ENV GOOS=linux
 # Changed from Apple Silicon arm64 config.
-ENV GOARCH=amd64
+# ENV GOARCH=amd64
-# ENV GOARCH=arm64
+ENV GOARCH=arm64
 
 WORKDIR /app
 

import.go

@@ -3,6 +3,7 @@ package main
 import (
 	"log"
 	"time"
+	"fmt"
 
 	"github.com/nprasad2077/NBA_Go/services"
 	"github.com/nprasad2077/NBA_Go/utils"
@@ -11,48 +12,48 @@ import (
 
 // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
 func importPlayerAdvanced(db *gorm.DB) {
-	for season := 2024; season <= 2025; season++ {
+	for season := 2003; season <= 2017; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
 		log.Printf("Advanced import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1000 * time.Millisecond)
 	}
 }
 
 // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
 func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 2024; season <= 2025; season++ {
+	for season := 2003; season <= 2017; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
 		log.Printf("Advanced Playoffs import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1500 * time.Millisecond)
 	}
 }
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-	for season := 2024; season <= 2025; season++ {
+	for season := 2003; season <= 2017; season++ {
 		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
 			log.Printf("scraped totals import failed for %d: %v", season, err)
 		}
 		log.Printf("Player Totals import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1250 * time.Millisecond)
 	}
 }
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-	for season := 2024; season <= 2025; season++ {
+	for season := 2003; season <= 2017; season++ {
 		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
 			log.Printf("scraped playoffs import failed for %d: %v", season, err)
 		}
 		log.Printf("Player Playoffs Totals import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1700 * time.Millisecond)
 	}
 }
@@ -67,7 +68,7 @@ func importGameSchedules(db *gorm.DB) {
 		// "february", "march", "april", "may", "june",
 	}
 
-	for season := 2025; season <= 2025; season++ {
+	for season := 2003; season <= 2008; season++ {
 		log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
 		for _, month := range months {
 			// The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished)
@@ -77,7 +78,7 @@ func importGameSchedules(db *gorm.DB) {
 			}
 			log.Printf("Game schedule import for %s, %d complete.", month, season)
 			// Respectful delay between requests
-			time.Sleep(1100 * time.Millisecond)
+			time.Sleep(1400 * time.Millisecond)
 			utils.SleepWithJitter(1800 * time.Millisecond)
 		}
 		log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
@@ -87,20 +88,24 @@ func importGameSchedules(db *gorm.DB) {
 // importBoxScores fetches and stores all box score data (line scores, player/team stats)
 // for games within a recent date range.
 func importBoxScores(db *gorm.DB) {
-    // Define the date range for the 2023-2024 NBA season.
+    from := time.Date(2025, time.February, 1, 0, 0, 0, 0, time.UTC)
-    // The regular season typically starts in October and playoffs end in June.
+    to := time.Date(2025, time.February, 14, 0, 0, 0, 0, time.UTC)
-    from := time.Date(2022, time.October, 1, 0, 0, 0, 0, time.UTC)
-    to := time.Date(2023, time.July, 1, 0, 0, 0, 0, time.UTC)
 
-    log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
+    dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", 
+        from.Format("January 2, 2006"), 
+        to.Format("January 2, 2006"))
+    
+    log.Println(dateRangeComment)
 
     if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
         log.Fatalf("Box score import failed: %v", err)
     }
 
-    log.Printf("--- Finished Box Score Data Import ---")
+    log.Println("--- Finished Box Score Data Import ---")
 }
 
+
+
 // importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
 func importPlayerShotCharts(db *gorm.DB) {
 	log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -117,21 +122,21 @@ func importPlayerShotCharts(db *gorm.DB) {
 	// Nikola Jokić:  "jokicni01"
 	// Luka Dončić:   "doncilu01"
 	targetPlayerIDs := []string{
-		"jamesle01",
+		// "jamesle01",
-		"curryst01",
+		// "curryst01",
-		"jokicni01",
+		// "jokicni01",
-		"doncilu01",
+		// "doncilu01",
-		"hardeja01",
+		// "hardeja01",
-		"irvinky01",
+		// "irvinky01",
-		"duranke01",
+		// "duranke01",
-		// "youngtr01",
+		"youngtr01",
-		// "lillada01",
+		"lillada01",
-		// "gilgesh01",
+		"gilgesh01",
-		// "brunsja01",
+		"brunsja01",
-		// "edwaran01",
+		"edwaran01",
-		// "mitchdo01",
+		"mitchdo01",
-		// "bookede01",
+		"bookede01",
-		// "derozde01",
+		"derozde01",
 
 		// Add more player IDs here as needed
 	}

main.go

@@ -39,9 +39,9 @@ import (
 
 	"github.com/nprasad2077/NBA_Go/config"
 	"github.com/nprasad2077/NBA_Go/controllers"
+	_ "github.com/nprasad2077/NBA_Go/docs"
 	"github.com/nprasad2077/NBA_Go/routes"
 	"github.com/nprasad2077/NBA_Go/utils/middleware"
-	_ "github.com/nprasad2077/NBA_Go/docs"
 )
 
 func main() {
@@ -65,12 +65,11 @@ func main() {
 		// importGameSchedules(db)
 		// log.Println("🎉 Game Imports completed successfully 🏀")
 
-		// importBoxScores(db)
+		importBoxScores(db)
-		// log.Println("🎉 Related Box Score Imports completed successfully 📦")
+		log.Println("🎉 Related Box Score Imports completed successfully 📦")
-
-		importPlayerShotCharts(db)
-		log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
 
+		// importPlayerShotCharts(db)
+		// log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
 
 		log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
 		return

services/box_score_scrape_service.go

@@ -16,7 +16,8 @@ import (
 )
 
 const boxScoreURLBase = "https://www.basketball-reference.com"
-const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network.
+const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network.
+const baseDelay = 3500 * time.Millisecond
 
 // ScrapedResult holds all the parsed stats from a single game.
 type ScrapedResult struct {
@@ -123,7 +124,8 @@ func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- Scrape
 		log.Printf("Worker %d: Processing game %s", id, game.GameID)
 		fullURL := boxScoreURLBase + game.BoxScoreURL
 
-		utils.SleepWithJitter(2300 * time.Millisecond)
+		utils.SleepWithJitter(baseDelay)
+		time.Sleep(1500 * time.Millisecond)
 
 		req, err := http.NewRequest("GET", fullURL, nil)
 		if err != nil {
@@ -332,38 +334,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
 	}
 }
 
+
 func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
+	const batchSize = 500 // 💡 A safe chunk size to stay under the limit.
+
+	// Batch upsert Player Basic Stats
 	if len(pbs) > 0 {
+		for i := 0; i < len(pbs); i += batchSize {
+			end := i + batchSize
+			if end > len(pbs) {
+				end = len(pbs)
+			}
+			batch := pbs[i:end]
 			if err := db.Clauses(clause.OnConflict{
 				Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
 				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
-		}).Create(&pbs).Error; err != nil {
+			}).Create(&batch).Error; err != nil {
-			return fmt.Errorf("failed to upsert player basic stats: %w", err)
+				return fmt.Errorf("failed to upsert player basic stats batch: %w", err)
 			}
 		}
+		log.Printf("Successfully upserted %d player basic stats.", len(pbs))
+	}
+
+	// Batch upsert Player Advanced Stats
 	if len(pas) > 0 {
+		for i := 0; i < len(pas); i += batchSize {
+			end := i + batchSize
+			if end > len(pas) {
+				end = len(pas)
+			}
+			batch := pas[i:end]
 			if err := db.Clauses(clause.OnConflict{
 				Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
 				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
-		}).Create(&pas).Error; err != nil {
+			}).Create(&batch).Error; err != nil {
-			return fmt.Errorf("failed to upsert player advanced stats: %w", err)
+				return fmt.Errorf("failed to upsert player advanced stats batch: %w", err)
 			}
 		}
+		log.Printf("Successfully upserted %d player advanced stats.", len(pas))
+	}
+
+	// Batch upsert Team Basic Stats
 	if len(tbs) > 0 {
+		for i := 0; i < len(tbs); i += batchSize {
+			end := i + batchSize
+			if end > len(tbs) {
+				end = len(tbs)
+			}
+			batch := tbs[i:end]
 			if err := db.Clauses(clause.OnConflict{
 				Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
 				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
-		}).Create(&tbs).Error; err != nil {
+			}).Create(&batch).Error; err != nil {
-			return fmt.Errorf("failed to upsert team basic stats: %w", err)
+				return fmt.Errorf("failed to upsert team basic stats batch: %w", err)
 			}
 		}
+		log.Printf("Successfully upserted %d team basic stats.", len(tbs))
+	}
+
+	// Batch upsert Team Advanced Stats
 	if len(tas) > 0 {
+		for i := 0; i < len(tas); i += batchSize {
+			end := i + batchSize
+			if end > len(tas) {
+				end = len(tas)
+			}
+			batch := tas[i:end]
 			if err := db.Clauses(clause.OnConflict{
 				Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
 				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
-		}).Create(&tas).Error; err != nil {
+			}).Create(&batch).Error; err != nil {
-			return fmt.Errorf("failed to upsert team advanced stats: %w", err)
+				return fmt.Errorf("failed to upsert team advanced stats batch: %w", err)
 			}
 		}
+		log.Printf("Successfully upserted %d team advanced stats.", len(tas))
+	}
+
 	return nil
 }