Merge pull request #42 from nprasad2077/games

Games

.gitignore

@@ -29,4 +29,6 @@ go.work.sum
 data/nba.db
 /data/
 /data
-/data/*.DS_Store
+/data/*
+docs/restore_dump_remote.md
+./docs/restore_dump_remote.md

Makefile

@@ -0,0 +1,10 @@
+# Tells 'make' these are command names, not files
+.PHONY: up down
+
+# Command to run: make up
+up:
+	docker-compose -f docker-compose.local.yml up --build -d
+
+# Command to run: make down
+down:
+	docker-compose down

README.md

@@ -1,6 +1,6 @@
 # NBA_Go
 
-### First‑time bootstrap
+## First‑time bootstrap
 
 ```bash
 # 1. build + run
@@ -18,19 +18,23 @@ curl http://localhost:8080/api/playeradvancedstats \
 
 ```
 
-### Swagger Initiate Docs
+## Swagger Initiate Docs
 
 ```bash
 swag init -g main.go -o docs
 ```
 
-### Test
+## Test
 
 ```bash
 go run loadtest.go -n 100 -c 10 -url "http://127.0.0.1:8080/api/playeradvancedstats?page=1&pageSize=20" -log results.log -key "xxx"
 ```
 
-### Local Environment
+## Local Environment
+
+```bash
+docker compose down
+```
 
 ```bash
 docker-compose -f docker-compose.local.yml up --build -d

docker-compose.local.yml

@@ -125,4 +125,4 @@ volumes:
   grafana_data:
 
 networks:
-  api.network: null
+  api.network: null

docs/postman/postman_docs.md

@@ -0,0 +1,24 @@
+# Run Collection Button Code
+
+## HTML
+
+```html
+<div class="postman-run-button"
+data-postman-action="collection/fork"
+data-postman-visibility="public"
+data-postman-var-1="25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6"
+data-postman-collection-url="entityId=25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6&entityType=collection&workspaceId=bb6fcb02-8388-4ce0-8623-63480a6dc153"></div>
+<script type="text/javascript">
+  (function (p,o,s,t,m,a,n) {
+    !p[s] && (p[s] = function () { (p[t] || (p[t] = [])).push(arguments); });
+    !o.getElementById(s+t) && o.getElementsByTagName("head")[0].appendChild((
+      (n = o.createElement("script")),
+      (n.id = s+t), (n.async = 1), (n.src = m), n
+    ));
+  }(window, document, "_pm", "PostmanRunObject", "https://run.pstmn.io/button.js"));
+</script>
+```
+
+## Markdown
+
+[<img src="https://run.pstmn.io/button.svg" alt="Run In Postman" style="width: 128px; height: 32px;">](https://god.gw.postman.com/run-collection/25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6%26entityType%3Dcollection%26workspaceId%3Dbb6fcb02-8388-4ce0-8623-63480a6dc153)

docs/restore_dump_local.md

@@ -0,0 +1,13 @@
+# Restore to local container
+
+```bash
+docker exec nba_go-postgres-1 pg_dump -U nbago -Fc nba_db > ./2025.12.05_nba_backup.dump
+```
+
+Copy file into target DB container
+
+```bash
+docker exec nba_postgres psql -U nba_admin -d nba_analytics -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
+
+pg_restore -U nba_admin -d nba_analytics --no-owner --no-acl ./2025.12.05_nba_backup.dump
+```

import.go

@@ -3,32 +3,33 @@ package main
 import (
 	"log"
 	"time"
+	"fmt"
 
 	"github.com/nprasad2077/NBA_Go/services"
 	"github.com/nprasad2077/NBA_Go/utils"
 	"gorm.io/gorm"
 )
 
-// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
+// importPlayerAdvanced fetches and stores advanced stats for seasons
 func importPlayerAdvanced(db *gorm.DB) {
 	for season := 2026; season <= 2026; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
 		log.Printf("Advanced import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1000 * time.Millisecond)
 	}
 }
 
-// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
+// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons
 func importPlayerAdvancedPlayoffs(db *gorm.DB) {
 	for season := 2026; season <= 2026; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
 		log.Printf("Advanced Playoffs import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1500 * time.Millisecond)
 	}
 }
@@ -40,7 +41,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
 			log.Printf("scraped totals import failed for %d: %v", season, err)
 		}
 		log.Printf("Player Totals import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1250 * time.Millisecond)
 	}
 }
@@ -52,7 +53,7 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
 			log.Printf("scraped playoffs import failed for %d: %v", season, err)
 		}
 		log.Printf("Player Playoffs Totals import for season: %d", season)
-		time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1300 * time.Millisecond)
 		utils.SleepWithJitter(1700 * time.Millisecond)
 	}
 }
@@ -61,12 +62,11 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
 func importGameSchedules(db *gorm.DB) {
 	// An NBA season typically runs from October to June
 	months := []string{
-		// "october", "november", "december", "january",
+		// "september", "october", "november", "december", "january",
 		// "february", "march", "april", "may", "june",
-
+		"january", "february",
-		"february", 
-
 		// "february", "march", "april", "may", "june",
+		// "october", "november", "december",
 	}
 
 	for season := 2025; season <= 2026; season++ {
@@ -79,7 +79,7 @@ func importGameSchedules(db *gorm.DB) {
 			}
 			log.Printf("Game schedule import for %s, %d complete.", month, season)
 			// Respectful delay between requests
-			time.Sleep(1100 * time.Millisecond)
+			time.Sleep(1400 * time.Millisecond)
 			utils.SleepWithJitter(1800 * time.Millisecond)
 		}
 		log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
@@ -89,20 +89,27 @@ func importGameSchedules(db *gorm.DB) {
 // importBoxScores fetches and stores all box score data (line scores, player/team stats)
 // for games within a recent date range.
 func importBoxScores(db *gorm.DB) {
-    // Define the date range for the 2023-2024 NBA season.
+	// now := time.Now()
-    // The regular season typically starts in October and playoffs end in June.
-    from := time.Date(2026, time.February, 1, 0, 0, 0, 0, time.UTC)
-    to := time.Date(2026, time.February, 10, 0, 0, 0, 0, time.UTC)
 
-    log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
+    from := time.Date(2026, time.January, 27, 0, 0, 0, 0, time.UTC)
+	// from := time.Date(now.Year(), now.Month(), now.Day()-1, 5, 30, 0, 0, time.UTC)
+    to := time.Date(2026, time.February, 28, 5, 30, 0, 0, time.UTC)
+
+    dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", 
+        from.Format("January 2, 2006"), 
+        to.Format("January 2, 2006"))
+    
+    log.Println(dateRangeComment)
 
     if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
         log.Fatalf("Box score import failed: %v", err)
     }
 
-    log.Printf("--- Finished Box Score Data Import ---")
+    log.Println("--- Finished Box Score Data Import ---")
 }
 
+
+
 // importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
 func importPlayerShotCharts(db *gorm.DB) {
 	log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -126,14 +133,22 @@ func importPlayerShotCharts(db *gorm.DB) {
 		"hardeja01",
 		"irvinky01",
 		"duranke01",
-		// "youngtr01",
+		"youngtr01",
 		// "lillada01",
-		// "gilgesh01",
+		"gilgesh01",
-		// "brunsja01",
+		"brunsja01",
-		// "edwaran01",
+		"edwaran01",
 		// "mitchdo01",
 		// "bookede01",
-		// "derozde01",
+		"derozde01",
+		// "aldrila01",
+		"westbru01",
+		// "paulch01",
+		"butleji01",
+		"davisan02",
+		"leonaka01",
+		// "youngtr01",
+		"tatumja01",
 
 		// Add more player IDs here as needed
 	}
@@ -160,4 +175,4 @@ func importPlayerShotCharts(db *gorm.DB) {
 	}
 
 	log.Println("--- Finished Player Shot Chart Import from Predefined List ---")
-}
+}

main.go

@@ -39,9 +39,9 @@ import (
 
 	"github.com/nprasad2077/NBA_Go/config"
 	"github.com/nprasad2077/NBA_Go/controllers"
+	_"github.com/nprasad2077/NBA_Go/docs"
 	"github.com/nprasad2077/NBA_Go/routes"
 	"github.com/nprasad2077/NBA_Go/utils/middleware"
-	_ "github.com/nprasad2077/NBA_Go/docs"
 )
 
 func main() {
@@ -53,8 +53,8 @@ func main() {
 		importPlayerAdvanced(db)
 		log.Println("🎉 Player Advanced Import completed successfully")
 
-		// importPlayerAdvancedPlayoffs(db)
+		// // importPlayerAdvancedPlayoffs(db)
-		// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
+		// // log.Println("🎉 Player Advanced Playoffs Import completed successfully")
 
 		importPlayerTotalsScrape(db)
 		log.Println("🎉 Player Totals (scraped) Import completed successfully")
@@ -71,7 +71,6 @@ func main() {
 		// importPlayerShotCharts(db)
 		// log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
 
-
 		log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
 		return
 	}
@@ -132,4 +131,4 @@ func main() {
 	log.Println("shutting down…")
 	_ = app.Shutdown()
 	log.Println("bye")
-}
+}

services/box_score_scrape_service.go

@@ -16,7 +16,8 @@ import (
 )
 
 const boxScoreURLBase = "https://www.basketball-reference.com"
-const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network.
+const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network.
+const baseDelay = 2500 * time.Millisecond
 
 // ScrapedResult holds all the parsed stats from a single game.
 type ScrapedResult struct {
@@ -119,11 +120,23 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro
 // scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
 func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
 	defer wg.Done()
+
+	// --- Stagger the start of each worker ---
+	// Calculate an offset based on the worker's ID to spread out the initial requests.
+	// We divide the base delay by the number of workers to get an even interval.
+	if numWorkers > 1 {
+		staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers))
+		initialDelay := time.Duration(id-1) * staggerAmount
+		log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay)
+		time.Sleep(initialDelay)
+	}
+
 	for game := range jobs {
-		log.Printf("Worker %d: Processing game %s", id, game.GameID)
+		log.Printf("🐝  Worker %d: Processing game %s", id, game.GameID)
 		fullURL := boxScoreURLBase + game.BoxScoreURL
 
-		utils.SleepWithJitter(2300 * time.Millisecond)
+		utils.SleepWithJitter(baseDelay)
+		time.Sleep(1500 * time.Millisecond)
 
 		req, err := http.NewRequest("GET", fullURL, nil)
 		if err != nil {
@@ -226,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
 			Q2:     mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
 			Q3:     mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
 			Q4:     mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
-			OT1:    mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
+			OT1:    mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()),
-			OT2:    mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
+			OT2:    mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()),
-			OT3:    mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
+			OT3:    mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()),
 			Total:  mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
 		})
 	})
@@ -332,38 +345,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
 	}
 }
 
+
 func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
+	const batchSize = 500 // 💡 A safe chunk size to stay under the limit.
+
+	// Batch upsert Player Basic Stats
 	if len(pbs) > 0 {
-		if err := db.Clauses(clause.OnConflict{
+		for i := 0; i < len(pbs); i += batchSize {
-			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			end := i + batchSize
-			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
+			if end > len(pbs) {
-		}).Create(&pbs).Error; err != nil {
+				end = len(pbs)
-			return fmt.Errorf("failed to upsert player basic stats: %w", err)
+			}
+			batch := pbs[i:end]
+			if err := db.Clauses(clause.OnConflict{
+				Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
+			}).Create(&batch).Error; err != nil {
+				return fmt.Errorf("failed to upsert player basic stats batch: %w", err)
+			}
 		}
+		log.Printf("Successfully upserted %d player basic stats.", len(pbs))
 	}
+
+	// Batch upsert Player Advanced Stats
 	if len(pas) > 0 {
-		if err := db.Clauses(clause.OnConflict{
+		for i := 0; i < len(pas); i += batchSize {
-			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			end := i + batchSize
-			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
+			if end > len(pas) {
-		}).Create(&pas).Error; err != nil {
+				end = len(pas)
-			return fmt.Errorf("failed to upsert player advanced stats: %w", err)
+			}
+			batch := pas[i:end]
+			if err := db.Clauses(clause.OnConflict{
+				Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
+			}).Create(&batch).Error; err != nil {
+				return fmt.Errorf("failed to upsert player advanced stats batch: %w", err)
+			}
 		}
+		log.Printf("Successfully upserted %d player advanced stats.", len(pas))
 	}
+
+	// Batch upsert Team Basic Stats
 	if len(tbs) > 0 {
-		if err := db.Clauses(clause.OnConflict{
+		for i := 0; i < len(tbs); i += batchSize {
-			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			end := i + batchSize
-			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
+			if end > len(tbs) {
-		}).Create(&tbs).Error; err != nil {
+				end = len(tbs)
-			return fmt.Errorf("failed to upsert team basic stats: %w", err)
+			}
+			batch := tbs[i:end]
+			if err := db.Clauses(clause.OnConflict{
+				Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
+			}).Create(&batch).Error; err != nil {
+				return fmt.Errorf("failed to upsert team basic stats batch: %w", err)
+			}
 		}
+		log.Printf("Successfully upserted %d team basic stats.", len(tbs))
 	}
+
+	// Batch upsert Team Advanced Stats
 	if len(tas) > 0 {
-		if err := db.Clauses(clause.OnConflict{
+		for i := 0; i < len(tas); i += batchSize {
-			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			end := i + batchSize
-			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
+			if end > len(tas) {
-		}).Create(&tas).Error; err != nil {
+				end = len(tas)
-			return fmt.Errorf("failed to upsert team advanced stats: %w", err)
+			}
+			batch := tas[i:end]
+			if err := db.Clauses(clause.OnConflict{
+				Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+				DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
+			}).Create(&batch).Error; err != nil {
+				return fmt.Errorf("failed to upsert team advanced stats batch: %w", err)
+			}
 		}
+		log.Printf("Successfully upserted %d team advanced stats.", len(tas))
 	}
+
 	return nil
 }

services/line_score_scrape_service.go

@@ -25,14 +25,22 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
 			GameID: gameID,
 			Team:   teamName,
 		}
+		
+		// Regular Quarters - no changes here
 		ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
 		ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
 		ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
 		ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
-		ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
+
-		ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
+		// --- FIX: Corrected selectors for Overtime periods ---
-		ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
+		// The HTML uses "1OT", "2OT", etc., not "OT1", "OT2".
+		ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
+		ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
+		ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
+		
+		// Total score
 		ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+		
 		lineScores = append(lineScores, ls)
 	})
 
@@ -42,13 +50,20 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
 	}
 
 	// Upsert the data into the database.
-	if err := db.Clauses(clause.OnConflict{
+	// Using a transaction is a good practice for atomicity.
-		Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+	err := db.Transaction(func(tx *gorm.DB) error {
-		DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+		return tx.Clauses(clause.OnConflict{
-	}).Create(&lineScores).Error; err != nil {
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+		}).Create(&lineScores).Error
+	})
+
+	if err != nil {
+		log.Printf("Error upserting line scores for game %s: %v", gameID, err)
 		return err
 	}
 	
 	log.Printf("Successfully processed line scores for game %s.", gameID)
 	return nil
-}
+}
+