| Game ID | ", + "Date | ", + "Visitor Team | ", + "Visitor Pts | ", + "Home Team | ", + "Home Pts | ", + "Arena | ", + "Duration | ", + "
|---|---|---|---|---|---|---|---|
| {{gameId}} | ", + "{{date}} | ", + "{{visitorTeam}} | ", + "{{visitorPts}} | ", + "{{homeTeam}} | ", + "{{homePts}} | ", + "{{arena}} | ", + "{{gameDuration}} | ", + "
| Game ID | ", + "Date | ", + "Visitor | ", + "Score | ", + "Home | ", + "Score | ", + "Arena | ", + "Duration | ", + "
|---|---|---|---|---|---|---|---|
| {{gameId}} | ", + "{{date}} | ", + "{{visitorTeam}} | ", + "{{visitorPts}} | ", + "{{homeTeam}} | ", + "{{homePts}} | ", + "{{arena}} | ", + "{{gameDuration}} | ", + "
| Team | ", + "Q1 | ", + "Q2 | ", + "Q3 | ", + "Q4 | ", + " {{#if hasOT}}", + "OT1 | ", + "OT2 | ", + "OT3 | ", + " {{/if}}", + "Total | ", + "
|---|---|---|---|---|---|---|---|---|
| {{team}} | ", + "{{q1}} | ", + "{{q2}} | ", + "{{q3}} | ", + "{{q4}} | ", + " {{#if ../hasOT}}", + "{{ot1}} | ", + "{{ot2}} | ", + "{{ot3}} | ", + " {{/if}}", + "{{total}} | ", + "
| Player | ", + "MIN | ", + "PTS | ", + "FG | ", + "FG% | ", + "3P | ", + "3P% | ", + "FT | ", + "FT% | ", + "REB | ", + "AST | ", + "STL | ", + "BLK | ", + "TOV | ", + "+/- | ", + "||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{playerName}} | ", + "Did Not Play | ", + " {{else}}", + "{{playerName}} | ", + "{{mp}} | ", + "{{pts}} | ", + "{{fg}}-{{fga}} | ", + "{{fgPct}} | ", + "{{threeP}}-{{threePa}} | ", + "{{threePPct}} | ", + "{{ft}}-{{fta}} | ", + "{{ftPct}} | ", + "{{trb}} | ", + "{{ast}} | ", + "{{stl}} | ", + "{{blk}} | ", + "{{tov}} | ", + "{{plusMinusDisplay}} | ", + " {{/if}}", + "
| ID | ", - "CreatedAt | ", - "UpdatedAt | ", - "DeletedAt | ", - "id | ", - "playerId | ", - "playerName | ", - "position | ", - "age | ", - "games | ", - "gamesStarted | ", - "minutesPg | ", - "fieldGoals | ", - "fieldAttempts | ", - "fieldPercent | ", - "threeFg | ", - "threeAttempts | ", - "threePercent | ", - "twoFg | ", - "twoAttempts | ", - "twoPercent | ", - "effectFgPercent | ", - "ft | ", - "ftAttempts | ", - "ftPercent | ", - "offensiveRb | ", - "defensiveRb | ", - "totalRb | ", - "assists | ", - "steals | ", - "blocks | ", - "turnovers | ", - "personalFouls | ", - "points | ", - "team | ", - "season | ", - "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{ID}} | ", - "{{CreatedAt}} | ", - "{{UpdatedAt}} | ", - "{{DeletedAt}} | ", - "{{id}} | ", - "{{playerId}} | ", - "{{playerName}} | ", - "{{position}} | ", - "{{age}} | ", - "{{games}} | ", - "{{gamesStarted}} | ", - "{{minutesPg}} | ", - "{{fieldGoals}} | ", - "{{fieldAttempts}} | ", - "{{fieldPercent}} | ", - "{{threeFg}} | ", - "{{threeAttempts}} | ", - "{{threePercent}} | ", - "{{twoFg}} | ", - "{{twoAttempts}} | ", - "{{twoPercent}} | ", - "{{effectFgPercent}} | ", - "{{ft}} | ", - "{{ftAttempts}} | ", - "{{ftPercent}} | ", - "{{offensiveRb}} | ", - "{{defensiveRb}} | ", - "{{totalRb}} | ", - "{{assists}} | ", - "{{steals}} | ", - "{{blocks}} | ", - "{{turnovers}} | ", - "{{personalFouls}} | ", - "{{points}} | ", - "{{team}} | ", - "{{season}} | ", - "
| Player | ", + "Pos | ", + "Games | ", + "Points | ", + "Assists | ", + "Rebounds | ", + "PPG | ", + "
|---|---|---|---|---|---|---|
| {{playerName}} | ", + "{{position}} | ", + "{{games}} | ", + "{{points}} | ", + "{{assists}} | ", + "{{totalRb}} | ", + "{{ppg}} | ", + "
| Player | ", + "Pos | ", + "Games | ", + "Points | ", + "Assists | ", + "Rebounds | ", + "PPG | ", + "
|---|---|---|---|---|---|---|
| {{playerName}} | ", + "{{position}} | ", + "{{games}} | ", + "{{points}} | ", + "{{assists}} | ", + "{{totalRb}} | ", + "{{ppg}} | ", + "
| Player | ", + "Pos | ", + "Games | ", + "Points | ", + "Assists | ", + "Rebounds | ", + "PPG | ", + "
|---|---|---|---|---|---|---|
| {{playerName}} | ", + "{{position}} | ", + "{{games}} | ", + "{{points}} | ", + "{{assists}} | ", + "{{totalRb}} | ", + "{{ppg}} | ", + "
| ID | ", - "CreatedAt | ", - "UpdatedAt | ", - "DeletedAt | ", - "id | ", - "playerId | ", - "playerName | ", - "position | ", - "age | ", - "games | ", - "gamesStarted | ", - "minutesPg | ", - "fieldGoals | ", - "fieldAttempts | ", - "fieldPercent | ", - "threeFg | ", - "threeAttempts | ", - "threePercent | ", - "twoFg | ", - "twoAttempts | ", - "twoPercent | ", - "effectFgPercent | ", - "ft | ", - "ftAttempts | ", - "ftPercent | ", - "offensiveRb | ", - "defensiveRb | ", - "totalRb | ", - "assists | ", - "steals | ", - "blocks | ", - "turnovers | ", - "personalFouls | ", - "points | ", - "team | ", - "season | ", - "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{ID}} | ", - "{{CreatedAt}} | ", - "{{UpdatedAt}} | ", - "{{DeletedAt}} | ", - "{{id}} | ", - "{{playerId}} | ", - "{{playerName}} | ", - "{{position}} | ", - "{{age}} | ", - "{{games}} | ", - "{{gamesStarted}} | ", - "{{minutesPg}} | ", - "{{fieldGoals}} | ", - "{{fieldAttempts}} | ", - "{{fieldPercent}} | ", - "{{threeFg}} | ", - "{{threeAttempts}} | ", - "{{threePercent}} | ", - "{{twoFg}} | ", - "{{twoAttempts}} | ", - "{{twoPercent}} | ", - "{{effectFgPercent}} | ", - "{{ft}} | ", - "{{ftAttempts}} | ", - "{{ftPercent}} | ", - "{{offensiveRb}} | ", - "{{defensiveRb}} | ", - "{{totalRb}} | ", - "{{assists}} | ", - "{{steals}} | ", - "{{blocks}} | ", - "{{turnovers}} | ", - "{{personalFouls}} | ", - "{{points}} | ", - "{{team}} | ", - "{{season}} | ", - "
| Player Name | ", + "Position | ", + "Games | ", + "Points | ", + "Assists | ", + "Rebounds | ", + "FG% | ", + "
|---|---|---|---|---|---|---|
| {{playerName}} | ", + "{{position}} | ", + "{{games}} | ", + "{{points}} | ", + "{{assists}} | ", + "{{totalRb}} | ", + "{{fieldPercent}} | ", + "
{{team}} - {{season}} {{playoffLabel}}
", + " ", + "Data sorted by Defensive Box +/- (descending)
", + "| Request | +Resp. time (Avg ms) | +90th (ms) | +95th (ms) | +99th (ms) | +Min (ms) | +Max (ms) | + + + +
|---|---|---|---|---|---|---|
|
+
+ GET
+ Games Detail
+
+
+ https://api.server.nbaapi.com/api/games?isPlayoff=false&page=1&pageSize=3&sortBy=date&ascending=false&include=lineScores%2C%20playerGameBasicStats
+ |
+
+ 533 | +1,215 | +1,489 | +2,637 | +57 | +9,117 | +
|
+
+ GET
+ Games
+
+
+ https://api.server.nbaapi.com/api/games?isPlayoff=false&page=1&pageSize=3&sortBy=date&ascending=false
+ |
+
+ 398 | +877 | +1,106 | +1,919 | +49 | +8,940 | +
|
+
+ GET
+ Player Totals Stats
+
+
+ {{base_url}}/api/playertotals?season=2026&team=HOU&page=1&pageSize=12&isPlayoff=False
+ |
+
+ 269 | +668 | +855 | +1,535 | +48 | +8,321 | +
|
+
+ GET
+ Player Totals Stats
+
+
+ {{base_url}}/api/playertotals?season=2026&team=HOU&page=1&pageSize=20&isPlayoff=False
+ |
+
+ 268 | +675 | +850 | +1,285 | +48 | +8,278 | +
|
+
+ GET
+ Player Totals Playoffs Stats
+
+
+ {{base_url}}/api/playertotals?team=HOU&page=1&pageSize=10&sortBy=assists&isPlayoff=true&season=2019
+ |
+
+ 266 | +643 | +856 | +1,429 | +48 | +8,715 | +
| Request | +Total requests | +Requests/s | +Min (ms) | +Avg (ms) | +90th (ms) | +Max (ms) | +Error % | + + + +
|---|---|---|---|---|---|---|---|
|
+
+ GET
+ Games
+
+ https://api.server.nbaapi.com/api/games?isPlayoff=false&page=1&pageSize=3&sortBy=date&ascending=false
+ |
+ 4,249 | +7.00 | +49 | +398 | +877 | +8,940 | +0 | +
|
+
+ GET
+ Games Copy
+
+ https://api.server.nbaapi.com/api/games?isPlayoff=true&page=1&pageSize=15&sortBy=date&ascending=true
+ |
+ 4,248 | +7.00 | +44 | +197 | +518 | +8,119 | +0 | +
|
+
+ GET
+ Games Detail
+
+ https://api.server.nbaapi.com/api/games?isPlayoff=false&page=1&pageSize=3&sortBy=date&ascending=false&include=lineScores%2C%20playerGameBasicStats
+ |
+ 4,242 | +6.99 | +57 | +533 | +1,215 | +9,117 | +0 | +
|
+
+ GET
+ Player Totals Stats
+
+ {{base_url}}/api/playertotals?season=2026&team=HOU&page=1&pageSize=12&isPlayoff=False
+ |
+ 4,240 | +6.99 | +48 | +269 | +668 | +8,321 | +0 | +
|
+
+ GET
+ Player Totals Playoffs Stats
+
+ {{base_url}}/api/playertotals?team=HOU&page=1&pageSize=10&sortBy=assists&isPlayoff=true&season=2019
+ |
+ 4,239 | +6.99 | +48 | +266 | +643 | +8,715 | +0 | +
|
+
+ GET
+ Player Advanced Stats
+
+ {{base_url}}/api/playeradvancedstats?season=2026&team=HOU&page=1&pageSize=4&ascending=false&isPlayoff=false
+ |
+ 4,238 | +6.99 | +46 | +260 | +669 | +2,742 | +0 | +
|
+
+ GET
+ Player Advanced Stats Copy
+
+ {{base_url}}/api/playeradvancedstats?season=2025&team=HOU&page=1&pageSize=3&ascending=false&isPlayoff=true
+ |
+ 4,234 | +6.98 | +47 | +263 | +670 | +8,492 | +0 | +
|
+
+ GET
+ Player Advanced Playoff Stats
+
+ {{base_url}}/api/playeradvancedstats?season=2025&team=BOS&page=1&pageSize=4&sortBy=winShares&ascending=false&isPlayoff=true
+ |
+ 4,233 | +6.98 | +48 | +248 | +644 | +3,454 | +0 | +
|
+
+ GET
+ Shot Chart
+
+ {{base_url}}/api/playershotchart?page=1&playerId=duranke01&date=Dec%2017%2C2018
+ |
+ 4,232 | +6.98 | +45 | +188 | +477 | +8,400 | +0 | +
|
+
+ GET
+ Shot Chart 2
+
+ {{base_url}}/api/playershotchart?page=1&playerId={{shot_player}}&season=2018&result=true&shot_type=3-pointer
+ |
+ 4,228 | +6.97 | +44 | +155 | +415 | +3,301 | +0 | +
|
+
+ GET
+ Player Advanced Stats
+
+ {{base_url}}/api/playeradvancedstats?season=2025&team=NYK&page=1&pageSize=4&sortBy=vorp&ascending=false&isPlayoff=true
+ |
+ 4,222 | +6.96 | +47 | +261 | +648 | +8,537 | +0 | +
|
+
+ GET
+ Player Totals Stats
+
+ {{base_url}}/api/playertotals?season=2026&team=HOU&page=1&pageSize=20&isPlayoff=False
+ |
+ 4,221 | +6.96 | +48 | +268 | +675 | +8,278 | +0 | +
+ 
](https://god.gw.postman.com/run-collection/25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6%26entityType%3Dcollection%26workspaceId%3Dbb6fcb02-8388-4ce0-8623-63480a6dc153)
diff --git a/docs/restore_dump_local.md b/docs/restore_dump_local.md
new file mode 100644
index 0000000..481de39
--- /dev/null
+++ b/docs/restore_dump_local.md
@@ -0,0 +1,13 @@
+# Restore to local container
+
+```bash
+docker exec nba_go-postgres-1 pg_dump -U nbago -Fc nba_db > ./2025.12.05_nba_backup.dump
+```
+
+Copy file into target DB container
+
+```bash
+docker exec nba_postgres psql -U nba_admin -d nba_analytics -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
+
+pg_restore -U nba_admin -d nba_analytics --no-owner --no-acl ./2025.12.05_nba_backup.dump
+```
diff --git a/import.go b/import.go
index 5ff1a3d..63c19a2 100644
--- a/import.go
+++ b/import.go
@@ -3,32 +3,33 @@ package main
import (
"log"
"time"
+ "fmt"
"github.com/nprasad2077/NBA_Go/services"
"github.com/nprasad2077/NBA_Go/utils"
"gorm.io/gorm"
)
-// importPlayerAdvanced fetches and stores advanced stats for seasons 2017β2025
+// importPlayerAdvanced fetches and stores advanced stats for seasons
func importPlayerAdvanced(db *gorm.DB) {
for season := 2026; season <= 2026; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced import for season: %d", season)
- time.Sleep(1100 * time.Millisecond)
+ time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
}
-// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023β2025
+// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2026; season <= 2026; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced Playoffs import for season: %d", season)
- time.Sleep(1100 * time.Millisecond)
+ time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1500 * time.Millisecond)
}
}
@@ -40,7 +41,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
log.Printf("Player Totals import for season: %d", season)
- time.Sleep(1100 * time.Millisecond)
+ time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1250 * time.Millisecond)
}
}
@@ -52,7 +53,7 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
log.Printf("Player Playoffs Totals import for season: %d", season)
- time.Sleep(1100 * time.Millisecond)
+ time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1700 * time.Millisecond)
}
}
@@ -61,12 +62,11 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
func importGameSchedules(db *gorm.DB) {
// An NBA season typically runs from October to June
months := []string{
- // "october", "november", "december", "january",
+ // "september", "october", "november", "december", "january",
// "february", "march", "april", "may", "june",
-
- "february",
-
+ "january", "february",
// "february", "march", "april", "may", "june",
+ // "october", "november", "december",
}
for season := 2025; season <= 2026; season++ {
@@ -79,7 +79,7 @@ func importGameSchedules(db *gorm.DB) {
}
log.Printf("Game schedule import for %s, %d complete.", month, season)
// Respectful delay between requests
- time.Sleep(1100 * time.Millisecond)
+ time.Sleep(1400 * time.Millisecond)
utils.SleepWithJitter(1800 * time.Millisecond)
}
log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
@@ -89,20 +89,27 @@ func importGameSchedules(db *gorm.DB) {
// importBoxScores fetches and stores all box score data (line scores, player/team stats)
// for games within a recent date range.
func importBoxScores(db *gorm.DB) {
- // Define the date range for the 2023-2024 NBA season.
- // The regular season typically starts in October and playoffs end in June.
- from := time.Date(2026, time.February, 1, 0, 0, 0, 0, time.UTC)
- to := time.Date(2026, time.February, 10, 0, 0, 0, 0, time.UTC)
+ // now := time.Now()
- log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
+ from := time.Date(2026, time.January, 27, 0, 0, 0, 0, time.UTC)
+ // from := time.Date(now.Year(), now.Month(), now.Day()-1, 5, 30, 0, 0, time.UTC)
+ to := time.Date(2026, time.February, 28, 5, 30, 0, 0, time.UTC)
+
+ dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---",
+ from.Format("January 2, 2006"),
+ to.Format("January 2, 2006"))
+
+ log.Println(dateRangeComment)
if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
log.Fatalf("Box score import failed: %v", err)
}
- log.Printf("--- Finished Box Score Data Import ---")
+ log.Println("--- Finished Box Score Data Import ---")
}
+
+
// importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
func importPlayerShotCharts(db *gorm.DB) {
log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -126,14 +133,22 @@ func importPlayerShotCharts(db *gorm.DB) {
"hardeja01",
"irvinky01",
"duranke01",
- // "youngtr01",
+ "youngtr01",
// "lillada01",
- // "gilgesh01",
- // "brunsja01",
- // "edwaran01",
+ "gilgesh01",
+ "brunsja01",
+ "edwaran01",
// "mitchdo01",
// "bookede01",
- // "derozde01",
+ "derozde01",
+ // "aldrila01",
+ "westbru01",
+ // "paulch01",
+ "butleji01",
+ "davisan02",
+ "leonaka01",
+ // "youngtr01",
+ "tatumja01",
// Add more player IDs here as needed
}
@@ -160,4 +175,4 @@ func importPlayerShotCharts(db *gorm.DB) {
}
log.Println("--- Finished Player Shot Chart Import from Predefined List ---")
-}
\ No newline at end of file
+}
diff --git a/main.go b/main.go
index 2e5c4da..646c148 100644
--- a/main.go
+++ b/main.go
@@ -39,9 +39,9 @@ import (
"github.com/nprasad2077/NBA_Go/config"
"github.com/nprasad2077/NBA_Go/controllers"
+ _"github.com/nprasad2077/NBA_Go/docs"
"github.com/nprasad2077/NBA_Go/routes"
"github.com/nprasad2077/NBA_Go/utils/middleware"
- _ "github.com/nprasad2077/NBA_Go/docs"
)
func main() {
@@ -53,8 +53,8 @@ func main() {
importPlayerAdvanced(db)
log.Println("π Player Advanced Import completed successfully")
- // importPlayerAdvancedPlayoffs(db)
- // log.Println("π Player Advanced Playoffs Import completed successfully")
+ // // importPlayerAdvancedPlayoffs(db)
+ // // log.Println("π Player Advanced Playoffs Import completed successfully")
importPlayerTotalsScrape(db)
log.Println("π Player Totals (scraped) Import completed successfully")
@@ -71,7 +71,6 @@ func main() {
// importPlayerShotCharts(db)
// log.Println("π Player Shot Chart Import completed successfully π―")
-
log.Println("π ALL Imports completed successfully β
π")
return
}
@@ -132,4 +131,4 @@ func main() {
log.Println("shutting downβ¦")
_ = app.Shutdown()
log.Println("bye")
-}
\ No newline at end of file
+}
diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go
index 67be818..2a453fd 100644
--- a/services/box_score_scrape_service.go
+++ b/services/box_score_scrape_service.go
@@ -16,7 +16,8 @@ import (
)
const boxScoreURLBase = "https://www.basketball-reference.com"
-const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network.
+const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network.
+const baseDelay = 2500 * time.Millisecond
// ScrapedResult holds all the parsed stats from a single game.
type ScrapedResult struct {
@@ -119,11 +120,23 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro
// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
defer wg.Done()
+
+ // --- Stagger the start of each worker ---
+ // Calculate an offset based on the worker's ID to spread out the initial requests.
+ // We divide the base delay by the number of workers to get an even interval.
+ if numWorkers > 1 {
+ staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers))
+ initialDelay := time.Duration(id-1) * staggerAmount
+ log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay)
+ time.Sleep(initialDelay)
+ }
+
for game := range jobs {
- log.Printf("Worker %d: Processing game %s", id, game.GameID)
+ log.Printf("π Worker %d: Processing game %s", id, game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
- utils.SleepWithJitter(2300 * time.Millisecond)
+ utils.SleepWithJitter(baseDelay)
+ time.Sleep(1500 * time.Millisecond)
req, err := http.NewRequest("GET", fullURL, nil)
if err != nil {
@@ -226,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
- OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
- OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
- OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
+ OT1: mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()),
+ OT2: mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()),
+ OT3: mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()),
Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
})
})
@@ -332,38 +345,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
}
}
+
func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
+ const batchSize = 500 // π‘ A safe chunk size to stay under the limit.
+
+ // Batch upsert Player Basic Stats
if len(pbs) > 0 {
- if err := db.Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
- DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
- }).Create(&pbs).Error; err != nil {
- return fmt.Errorf("failed to upsert player basic stats: %w", err)
+ for i := 0; i < len(pbs); i += batchSize {
+ end := i + batchSize
+ if end > len(pbs) {
+ end = len(pbs)
+ }
+ batch := pbs[i:end]
+ if err := db.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+ DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
+ }).Create(&batch).Error; err != nil {
+ return fmt.Errorf("failed to upsert player basic stats batch: %w", err)
+ }
}
+ log.Printf("Successfully upserted %d player basic stats.", len(pbs))
}
+
+ // Batch upsert Player Advanced Stats
if len(pas) > 0 {
- if err := db.Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
- DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
- }).Create(&pas).Error; err != nil {
- return fmt.Errorf("failed to upsert player advanced stats: %w", err)
+ for i := 0; i < len(pas); i += batchSize {
+ end := i + batchSize
+ if end > len(pas) {
+ end = len(pas)
+ }
+ batch := pas[i:end]
+ if err := db.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+ DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
+ }).Create(&batch).Error; err != nil {
+ return fmt.Errorf("failed to upsert player advanced stats batch: %w", err)
+ }
}
+ log.Printf("Successfully upserted %d player advanced stats.", len(pas))
}
+
+ // Batch upsert Team Basic Stats
if len(tbs) > 0 {
- if err := db.Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
- DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
- }).Create(&tbs).Error; err != nil {
- return fmt.Errorf("failed to upsert team basic stats: %w", err)
+ for i := 0; i < len(tbs); i += batchSize {
+ end := i + batchSize
+ if end > len(tbs) {
+ end = len(tbs)
+ }
+ batch := tbs[i:end]
+ if err := db.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
+ DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
+ }).Create(&batch).Error; err != nil {
+ return fmt.Errorf("failed to upsert team basic stats batch: %w", err)
+ }
}
+ log.Printf("Successfully upserted %d team basic stats.", len(tbs))
}
+
+ // Batch upsert Team Advanced Stats
if len(tas) > 0 {
- if err := db.Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
- DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
- }).Create(&tas).Error; err != nil {
- return fmt.Errorf("failed to upsert team advanced stats: %w", err)
+ for i := 0; i < len(tas); i += batchSize {
+ end := i + batchSize
+ if end > len(tas) {
+ end = len(tas)
+ }
+ batch := tas[i:end]
+ if err := db.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
+ DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
+ }).Create(&batch).Error; err != nil {
+ return fmt.Errorf("failed to upsert team advanced stats batch: %w", err)
+ }
}
+ log.Printf("Successfully upserted %d team advanced stats.", len(tas))
}
+
return nil
}
\ No newline at end of file
diff --git a/services/line_score_scrape_service.go b/services/line_score_scrape_service.go
index 7f6f4a4..0d5b35c 100644
--- a/services/line_score_scrape_service.go
+++ b/services/line_score_scrape_service.go
@@ -25,14 +25,22 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
GameID: gameID,
Team: teamName,
}
+
+ // Regular Quarters - no changes here
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
- ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
- ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
- ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
+
+ // --- FIX: Corrected selectors for Overtime periods ---
+ // The HTML uses "1OT", "2OT", etc., not "OT1", "OT2".
+ ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
+ ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
+ ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
+
+ // Total score
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+
lineScores = append(lineScores, ls)
})
@@ -42,13 +50,20 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
}
// Upsert the data into the database.
- if err := db.Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
- DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
- }).Create(&lineScores).Error; err != nil {
+ // Using a transaction is a good practice for atomicity.
+ err := db.Transaction(func(tx *gorm.DB) error {
+ return tx.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
+ DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+ }).Create(&lineScores).Error
+ })
+
+ if err != nil {
+ log.Printf("Error upserting line scores for game %s: %v", gameID, err)
return err
}
log.Printf("Successfully processed line scores for game %s.", gameID)
return nil
-}
\ No newline at end of file
+}
+