diff --git a/.DS_Store b/.DS_Store index bd3a02a..146dc2b 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/Dockerfile b/Dockerfile index 9f0af2e..0700431 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ FROM golang:1.24-bullseye AS builder ENV CGO_ENABLED=1 ENV GOOS=linux # Changed from Apple Silicon arm64 config. -ENV GOARCH=amd64 -# ENV GOARCH=arm64 +# ENV GOARCH=amd64 +ENV GOARCH=arm64 WORKDIR /app diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 847d123..fafea45 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -125,4 +125,4 @@ volumes: grafana_data: networks: - api.network: null \ No newline at end of file + api.network: null diff --git a/import.go b/import.go index ff7d9e4..e263ba6 100644 --- a/import.go +++ b/import.go @@ -3,6 +3,7 @@ package main import ( "log" "time" + "fmt" "github.com/nprasad2077/NBA_Go/services" "github.com/nprasad2077/NBA_Go/utils" @@ -11,48 +12,48 @@ import ( // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 func importPlayerAdvanced(db *gorm.DB) { - for season := 2024; season <= 2025; season++ { + for season := 2003; season <= 2017; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } log.Printf("Advanced import for season: %d", season) - time.Sleep(1100 * time.Millisecond) + time.Sleep(1300 * time.Millisecond) utils.SleepWithJitter(1000 * time.Millisecond) } } // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 func importPlayerAdvancedPlayoffs(db *gorm.DB) { - for season := 2024; season <= 2025; season++ { + for season := 2003; season <= 2017; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } log.Printf("Advanced Playoffs import for season: %d", season) - time.Sleep(1100 * time.Millisecond) + time.Sleep(1300 * time.Millisecond) utils.SleepWithJitter(1500 * time.Millisecond) } } // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 2024; season <= 2025; season++ { + for season := 2003; season <= 2017; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } log.Printf("Player Totals import for season: %d", season) - time.Sleep(1100 * time.Millisecond) + time.Sleep(1300 * time.Millisecond) utils.SleepWithJitter(1250 * time.Millisecond) } } // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 2024; season <= 2025; season++ { + for season := 2003; season <= 2017; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } log.Printf("Player Playoffs Totals import for season: %d", season) - time.Sleep(1100 * time.Millisecond) + time.Sleep(1300 * time.Millisecond) utils.SleepWithJitter(1700 * time.Millisecond) } } @@ -67,7 +68,7 @@ func importGameSchedules(db *gorm.DB) { // "february", "march", "april", "may", "june", } - for season := 2025; season <= 2025; season++ { + for season := 2003; season <= 2008; season++ { log.Printf("--- Starting Game Schedule Import for Season: %d ---", season) for _, month := range months { // The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished) @@ -77,7 +78,7 @@ func importGameSchedules(db *gorm.DB) { } log.Printf("Game schedule import for %s, %d complete.", month, season) // Respectful delay between requests - time.Sleep(1100 * time.Millisecond) + time.Sleep(1400 * time.Millisecond) utils.SleepWithJitter(1800 * time.Millisecond) } log.Printf("--- Finished Game Schedule Import for Season: %d ---", season) @@ -87,20 +88,24 @@ func importGameSchedules(db *gorm.DB) { // importBoxScores fetches and stores all box score data (line scores, player/team stats) // for games within a recent date range. func importBoxScores(db *gorm.DB) { - // Define the date range for the 2023-2024 NBA season. - // The regular season typically starts in October and playoffs end in June. - from := time.Date(2022, time.October, 1, 0, 0, 0, 0, time.UTC) - to := time.Date(2023, time.July, 1, 0, 0, 0, 0, time.UTC) + from := time.Date(2025, time.February, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2025, time.February, 14, 0, 0, 0, 0, time.UTC) - log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---") + dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", + from.Format("January 2, 2006"), + to.Format("January 2, 2006")) + + log.Println(dateRangeComment) if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { log.Fatalf("Box score import failed: %v", err) } - log.Printf("--- Finished Box Score Data Import ---") + log.Println("--- Finished Box Score Data Import ---") } + + // importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons. func importPlayerShotCharts(db *gorm.DB) { log.Println("--- Starting Player Shot Chart Import from Predefined List ---") @@ -117,21 +122,21 @@ func importPlayerShotCharts(db *gorm.DB) { // Nikola Jokić: "jokicni01" // Luka Dončić: "doncilu01" targetPlayerIDs := []string{ - "jamesle01", - "curryst01", - "jokicni01", - "doncilu01", - "hardeja01", - "irvinky01", - "duranke01", - // "youngtr01", - // "lillada01", - // "gilgesh01", - // "brunsja01", - // "edwaran01", - // "mitchdo01", - // "bookede01", - // "derozde01", + // "jamesle01", + // "curryst01", + // "jokicni01", + // "doncilu01", + // "hardeja01", + // "irvinky01", + // "duranke01", + "youngtr01", + "lillada01", + "gilgesh01", + "brunsja01", + "edwaran01", + "mitchdo01", + "bookede01", + "derozde01", // Add more player IDs here as needed } @@ -158,4 +163,4 @@ func importPlayerShotCharts(db *gorm.DB) { } log.Println("--- Finished Player Shot Chart Import from Predefined List ---") -} \ No newline at end of file +} diff --git a/main.go b/main.go index 7ae1302..535b39b 100644 --- a/main.go +++ b/main.go @@ -39,9 +39,9 @@ import ( "github.com/nprasad2077/NBA_Go/config" "github.com/nprasad2077/NBA_Go/controllers" + _ "github.com/nprasad2077/NBA_Go/docs" "github.com/nprasad2077/NBA_Go/routes" "github.com/nprasad2077/NBA_Go/utils/middleware" - _ "github.com/nprasad2077/NBA_Go/docs" ) func main() { @@ -65,12 +65,11 @@ func main() { // importGameSchedules(db) // log.Println("🎉 Game Imports completed successfully 🏀") - // importBoxScores(db) - // log.Println("🎉 Related Box Score Imports completed successfully 📦") - - importPlayerShotCharts(db) - log.Println("🎉 Player Shot Chart Import completed successfully 🎯") + importBoxScores(db) + log.Println("🎉 Related Box Score Imports completed successfully 📦") + // importPlayerShotCharts(db) + // log.Println("🎉 Player Shot Chart Import completed successfully 🎯") log.Println("🏀 ALL Imports completed successfully ✅ 🙌") return @@ -132,4 +131,4 @@ func main() { log.Println("shutting down…") _ = app.Shutdown() log.Println("bye") -} \ No newline at end of file +} diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go index 67be818..81f124d 100644 --- a/services/box_score_scrape_service.go +++ b/services/box_score_scrape_service.go @@ -16,7 +16,8 @@ import ( ) const boxScoreURLBase = "https://www.basketball-reference.com" -const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network. +const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network. +const baseDelay = 3500 * time.Millisecond // ScrapedResult holds all the parsed stats from a single game. type ScrapedResult struct { @@ -123,7 +124,8 @@ func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- Scrape log.Printf("Worker %d: Processing game %s", id, game.GameID) fullURL := boxScoreURLBase + game.BoxScoreURL - utils.SleepWithJitter(2300 * time.Millisecond) + utils.SleepWithJitter(baseDelay) + time.Sleep(1500 * time.Millisecond) req, err := http.NewRequest("GET", fullURL, nil) if err != nil { @@ -332,38 +334,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa } } + func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error { + const batchSize = 500 // 💡 A safe chunk size to stay under the limit. + + // Batch upsert Player Basic Stats if len(pbs) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})), - }).Create(&pbs).Error; err != nil { - return fmt.Errorf("failed to upsert player basic stats: %w", err) + for i := 0; i < len(pbs); i += batchSize { + end := i + batchSize + if end > len(pbs) { + end = len(pbs) + } + batch := pbs[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert player basic stats batch: %w", err) + } } + log.Printf("Successfully upserted %d player basic stats.", len(pbs)) } + + // Batch upsert Player Advanced Stats if len(pas) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})), - }).Create(&pas).Error; err != nil { - return fmt.Errorf("failed to upsert player advanced stats: %w", err) + for i := 0; i < len(pas); i += batchSize { + end := i + batchSize + if end > len(pas) { + end = len(pas) + } + batch := pas[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert player advanced stats batch: %w", err) + } } + log.Printf("Successfully upserted %d player advanced stats.", len(pas)) } + + // Batch upsert Team Basic Stats if len(tbs) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})), - }).Create(&tbs).Error; err != nil { - return fmt.Errorf("failed to upsert team basic stats: %w", err) + for i := 0; i < len(tbs); i += batchSize { + end := i + batchSize + if end > len(tbs) { + end = len(tbs) + } + batch := tbs[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert team basic stats batch: %w", err) + } } + log.Printf("Successfully upserted %d team basic stats.", len(tbs)) } + + // Batch upsert Team Advanced Stats if len(tas) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})), - }).Create(&tas).Error; err != nil { - return fmt.Errorf("failed to upsert team advanced stats: %w", err) + for i := 0; i < len(tas); i += batchSize { + end := i + batchSize + if end > len(tas) { + end = len(tas) + } + batch := tas[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert team advanced stats batch: %w", err) + } } + log.Printf("Successfully upserted %d team advanced stats.", len(tas)) } + return nil } \ No newline at end of file