diff --git a/.DS_Store b/.DS_Store index 146dc2b..0960bde 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/import.go b/import.go index e263ba6..df515a4 100644 --- a/import.go +++ b/import.go @@ -62,13 +62,13 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { func importGameSchedules(db *gorm.DB) { // An NBA season typically runs from October to June months := []string{ - "october", "november", "december", "january", + "septempber", "october", "november", "december", "january", "february", "march", "april", "may", "june", // "february", "march", "april", "may", "june", } - for season := 2003; season <= 2008; season++ { + for season := 2024; season <= 2026; season++ { log.Printf("--- Starting Game Schedule Import for Season: %d ---", season) for _, month := range months { // The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished) @@ -88,8 +88,8 @@ func importGameSchedules(db *gorm.DB) { // importBoxScores fetches and stores all box score data (line scores, player/team stats) // for games within a recent date range. func importBoxScores(db *gorm.DB) { - from := time.Date(2025, time.February, 1, 0, 0, 0, 0, time.UTC) - to := time.Date(2025, time.February, 14, 0, 0, 0, 0, time.UTC) + from := time.Date(2025, time.October, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", from.Format("January 2, 2006"), @@ -129,14 +129,22 @@ func importPlayerShotCharts(db *gorm.DB) { // "hardeja01", // "irvinky01", // "duranke01", + // "youngtr01", + // "lillada01", + // "gilgesh01", + // "brunsja01", + // "edwaran01", + // "mitchdo01", + // "bookede01", + // "derozde01", + // "aldrila01", + // "westbru01", + // "paulch01", + // "butleji01", + "davisan02", + "leonaka01", "youngtr01", - "lillada01", - "gilgesh01", - "brunsja01", - "edwaran01", - "mitchdo01", - "bookede01", - "derozde01", + "tatumja01", // Add more player IDs here as needed } diff --git a/main.go b/main.go index 535b39b..2da8e04 100644 --- a/main.go +++ b/main.go @@ -62,8 +62,8 @@ func main() { // importPlayerTotalsPlayoffsScrape(db) // log.Println("🎉 Player Playoffs (scraped) Import completed successfully") - // importGameSchedules(db) - // log.Println("🎉 Game Imports completed successfully 🏀") + importGameSchedules(db) + log.Println("🎉 Game Imports completed successfully 🏀") importBoxScores(db) log.Println("🎉 Related Box Score Imports completed successfully 📦") diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go index 81f124d..a4111cc 100644 --- a/services/box_score_scrape_service.go +++ b/services/box_score_scrape_service.go @@ -16,8 +16,8 @@ import ( ) const boxScoreURLBase = "https://www.basketball-reference.com" -const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network. -const baseDelay = 3500 * time.Millisecond +const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network. +const baseDelay = 2500 * time.Millisecond // ScrapedResult holds all the parsed stats from a single game. type ScrapedResult struct { @@ -120,6 +120,17 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro // scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result. func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) { defer wg.Done() + + // --- Stagger the start of each worker --- + // Calculate an offset based on the worker's ID to spread out the initial requests. + // We divide the base delay by the number of workers to get an even interval. + if numWorkers > 1 { + staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers)) + initialDelay := time.Duration(id-1) * staggerAmount + log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay) + time.Sleep(initialDelay) + } + for game := range jobs { log.Printf("Worker %d: Processing game %s", id, game.GameID) fullURL := boxScoreURLBase + game.BoxScoreURL @@ -228,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore { Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()), Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()), Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()), - OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()), - OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()), - OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()), + OT1: mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()), + OT2: mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()), + OT3: mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()), Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()), }) }) diff --git a/services/line_score_scrape_service.go b/services/line_score_scrape_service.go index 7f6f4a4..0d5b35c 100644 --- a/services/line_score_scrape_service.go +++ b/services/line_score_scrape_service.go @@ -25,14 +25,22 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e GameID: gameID, Team: teamName, } + + // Regular Quarters - no changes here ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text()) ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text()) ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text()) ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text()) - ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()) - ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()) - ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()) + + // --- FIX: Corrected selectors for Overtime periods --- + // The HTML uses "1OT", "2OT", etc., not "OT1", "OT2". + ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()) + ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()) + ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()) + + // Total score ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text()) + lineScores = append(lineScores, ls) }) @@ -42,13 +50,20 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e } // Upsert the data into the database. - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}), - }).Create(&lineScores).Error; err != nil { + // Using a transaction is a good practice for atomicity. + err := db.Transaction(func(tx *gorm.DB) error { + return tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}), + }).Create(&lineScores).Error + }) + + if err != nil { + log.Printf("Error upserting line scores for game %s: %v", gameID, err) return err } log.Printf("Successfully processed line scores for game %s.", gameID) return nil -} \ No newline at end of file +} +