diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 0968822..847d123 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -25,7 +25,7 @@ services: db-init: build: . env_file: .env - command: ["/nba_go"] + command: ["/nba_go", "import-data"] depends_on: postgres: condition: service_healthy diff --git a/import.go b/import.go index 13ca0ac..45b0e55 100644 --- a/import.go +++ b/import.go @@ -87,20 +87,18 @@ func importGameSchedules(db *gorm.DB) { // importBoxScores fetches and stores all box score data (line scores, player/team stats) // for games within a recent date range. func importBoxScores(db *gorm.DB) { - // Define the date range for the import. + // Define the date range for the 2023-2024 NBA season. + // The regular season typically starts in October and playoffs end in June. + from := time.Date(2023, time.October, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2024, time.July, 1, 0, 0, 0, 0, time.UTC) - // The format is: time.Date(year, month, day, hour, min, sec, nsec, location) - // to := time.Date(2019, time.June, 14, 0, 0, 0, 0, time.UTC) - to := time.Now() - from := to.AddDate(-1, 0, 0) // 0 years, -3 months, 0 days + log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---") - log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02")) + if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { + log.Fatalf("Box score import failed: %v", err) + } - if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { - log.Fatalf("Box score import failed: %v", err) - } - - log.Printf("--- Finished Box Score Data Import ---") + log.Printf("--- Finished Box Score Data Import ---") } // importPlayerShotChart fetches shot-charts for every known player diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go index 5098263..98ee511 100644 --- a/services/box_score_scrape_service.go +++ b/services/box_score_scrape_service.go @@ -5,6 +5,7 @@ import ( "log" "net/http" "strings" + "sync" "time" "github.com/PuerkitoBio/goquery" @@ -15,149 +16,229 @@ import ( ) const boxScoreURLBase = "https://www.basketball-reference.com" +const numWorkers = 8 // Number of concurrent scrapers. Adjust based on your machine and network. + +// ScrapedResult holds all the parsed stats from a single game. +type ScrapedResult struct { + PlayerBasicStats []models.PlayerGameBasicStat + PlayerAdvStats []models.PlayerGameAdvStat + TeamBasicStats []models.TeamGameBasicStat + TeamAdvStats []models.TeamGameAdvStat + LineScores []models.LineScore + GameID string + Err error +} // uncommentDoc finds and replaces commented out HTML sections. func uncommentDoc(doc *goquery.Document) *goquery.Document { doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool { return goquery.NodeName(s) == "#comment" }).Each(func(i int, s *goquery.Selection) { - // Use .Data on the underlying html.Node to get the comment content. commentText := s.Nodes[0].Data if strings.Contains(commentText, "= ? AND date < ?", from, to).Find(&games).Error; err != nil { + if err := db.Where("date >= ? AND date < ?", from, to.Add(24*time.Hour)).Find(&games).Error; err != nil { return fmt.Errorf("failed to query games from DB: %w", err) } - log.Printf("Found %d games to process in the specified date range.", len(games)) + if len(games) == 0 { + log.Println("No games found to process in the specified date range.") + return nil + } + log.Printf("Found %d games to process. Initializing concurrent scraping...", len(games)) + // --- Concurrency Setup --- + jobs := make(chan models.Game, len(games)) + results := make(chan ScrapedResult, len(games)) + var wg sync.WaitGroup + + // Start worker goroutines + for w := 1; w <= numWorkers; w++ { + wg.Add(1) + go scrapeAndParseWorker(w, jobs, results, &wg) + } + + // Send jobs to the workers for _, game := range games { - log.Printf("Processing game: %s", game.GameID) - fullURL := boxScoreURLBase + game.BoxScoreURL - if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil { - // Log the error but continue to the next game - log.Printf("Error processing box score for game %s: %v", game.GameID, err) + jobs <- game + } + close(jobs) + + // Wait for all workers to finish + wg.Wait() + close(results) + + // --- Aggregation & Final Upsert --- + log.Println("All scraping complete. Aggregating results for final batch upsert...") + var allPlayerBasicStats []models.PlayerGameBasicStat + var allPlayerAdvStats []models.PlayerGameAdvStat + var allTeamBasicStats []models.TeamGameBasicStat + var allTeamAdvStats []models.TeamGameAdvStat + var allLineScores []models.LineScore + + for res := range results { + if res.Err != nil { + log.Printf("A worker failed on game %s: %v", res.GameID, res.Err) + continue } - // Be a good internet citizen and pause between requests. + allPlayerBasicStats = append(allPlayerBasicStats, res.PlayerBasicStats...) + allPlayerAdvStats = append(allPlayerAdvStats, res.PlayerAdvStats...) + allTeamBasicStats = append(allTeamBasicStats, res.TeamBasicStats...) + allTeamAdvStats = append(allTeamAdvStats, res.TeamAdvStats...) + allLineScores = append(allLineScores, res.LineScores...) + } + + // Upsert Line Scores first + if len(allLineScores) > 0 { + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.LineScore{})), + }).Create(&allLineScores).Error; err != nil { + return fmt.Errorf("failed to upsert line scores: %w", err) + } + log.Printf("Successfully upserted %d line scores.", len(allLineScores)) + } + + // Call the batch upsert function with the fully aggregated data + if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil { + return fmt.Errorf("final batch upsert failed: %w", err) + } + + log.Printf("Successfully upserted all box score data for %d games.", len(games)) + return nil +} + +// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result. +func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) { + defer wg.Done() + for game := range jobs { + log.Printf("Worker %d: Processing game %s", id, game.GameID) + fullURL := boxScoreURLBase + game.BoxScoreURL + utils.SleepWithJitter(2300 * time.Millisecond) + + req, err := http.NewRequest("GET", fullURL, nil) + if err != nil { + results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("failed to create request: %w", err)} + continue + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + resp, err := http.DefaultClient.Do(req) + if err != nil { + results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("request failed: %w", err)} + continue + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("received non-200 status code: %s", resp.Status)} + continue + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + resp.Body.Close() + if err != nil { + results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("failed to parse document: %w", err)} + continue + } + + doc = uncommentDoc(doc) + + lineScores := parseLineScore(doc, game.GameID) + pbs, pas, tbs, tas := parseBoxScores(doc, game.GameID) + + results <- ScrapedResult{ + PlayerBasicStats: pbs, + PlayerAdvStats: pas, + TeamBasicStats: tbs, + TeamAdvStats: tas, + LineScores: lineScores, + GameID: game.GameID, + Err: nil, + } } - return nil } -// scrapeBoxScorePage handles fetching and parsing a single box score page. -func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("received non-200 status code: %s", resp.Status) - } - - doc, err := goquery.NewDocumentFromReader(resp.Body) - if err != nil { - return err - } - - // 1. Uncomment all tables in the document first. - doc = uncommentDoc(doc) - - // 2. Call the dedicated service to handle line scores. - if err := FetchAndStoreLineScore(db, doc, gameID); err != nil { - log.Printf("Error processing line score for game %s: %v", gameID, err) - } - - // 3. The existing box score parser will now work because its tables are visible. - if err := parseAndStoreBoxScores(db, doc, gameID); err != nil { - return err - } - - return nil -} - -// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them. -func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error { +// parseBoxScores now returns the slices instead of calling the DB. +func parseBoxScores(doc *goquery.Document, gameID string) ([]models.PlayerGameBasicStat, []models.PlayerGameAdvStat, []models.TeamGameBasicStat, []models.TeamGameAdvStat) { var allPlayerBasicStats []models.PlayerGameBasicStat var allPlayerAdvStats []models.PlayerGameAdvStat var allTeamBasicStats []models.TeamGameBasicStat var allTeamAdvStats []models.TeamGameAdvStat - // Use a CSS attribute selector to find all box score tables for both teams. doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) { tableID, _ := table.Attr("id") isAdvanced := strings.Contains(tableID, "-advanced") + teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic") teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced") - // Process player rows table.Find("tbody tr").Each(func(j int, row *goquery.Selection) { playerID, exists := row.Find("th").Attr("data-append-csv") if !exists || playerID == "" { - return // Not a player row + return } - // Handle "Did Not Play" or other statuses - reason := row.Find(`td[data-stat="reason"]`) status := "Played" - if reason.Length() > 0 { + if reason := row.Find(`td[data-stat="reason"]`); reason.Length() > 0 { status = reason.Text() } if !isAdvanced { - stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status) - allPlayerBasicStats = append(allPlayerBasicStats, stat) + allPlayerBasicStats = append(allPlayerBasicStats, parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status)) } else { - stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status) - allPlayerAdvStats = append(allPlayerAdvStats, stat) + allPlayerAdvStats = append(allPlayerAdvStats, parsePlayerAdvStat(row, gameID, playerID, teamAbbr)) } }) - // Process team total row table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) { if !isAdvanced { - stat := parseTeamBasicStat(row, gameID, teamAbbr) - allTeamBasicStats = append(allTeamBasicStats, stat) + allTeamBasicStats = append(allTeamBasicStats, parseTeamBasicStat(row, gameID, teamAbbr)) } else { - stat := parseTeamAdvStat(row, gameID, teamAbbr) - allTeamAdvStats = append(allTeamAdvStats, stat) + allTeamAdvStats = append(allTeamAdvStats, parseTeamAdvStat(row, gameID, teamAbbr)) } }) }) - // Batch upsert all collected stats - if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil { - return err - } - - return nil + return allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats } -// --- Parsing Helper Functions --- +func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore { + var lineScores []models.LineScore + doc.Find("#line_score tbody tr").Each(func(i int, row *goquery.Selection) { + teamAbbr := row.Find(`th a`).Text() + if teamAbbr == "" { + return + } + + lineScores = append(lineScores, models.LineScore{ + GameID: gameID, + Team: teamAbbr, + Q1: mustAtoi(row.Find(`td[data-stat="1"]`).Text()), + Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()), + Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()), + Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()), + OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()), + OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()), + OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()), + Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()), + }) + }) + return lineScores +} func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat { return models.PlayerGameBasicStat{ - GameID: gameID, - PlayerID: playerID, + GameID: gameID, PlayerID: playerID, Team: team, Status: status, PlayerName: row.Find(`th[data-stat="player"] a`).Text(), - Team: team, - Status: status, MP: row.Find(`td[data-stat="mp"]`).Text(), FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()), FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()), @@ -178,16 +259,14 @@ func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status PF: mustAtoi(row.Find(`td[data-stat="pf"]`).Text()), PTS: mustAtoi(row.Find(`td[data-stat="pts"]`).Text()), GmSc: mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()), - PlusMinus: mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()), + PlusMinus: mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()), // <-- UPDATED LINE } } -func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat { +func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team string) models.PlayerGameAdvStat { return models.PlayerGameAdvStat{ - GameID: gameID, - PlayerID: playerID, + GameID: gameID, PlayerID: playerID, Team: team, PlayerName: row.Find(`th[data-stat="player"] a`).Text(), - Team: team, MP: row.Find(`td[data-stat="mp"]`).Text(), TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()), EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()), @@ -209,8 +288,7 @@ func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status s func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat { return models.TeamGameBasicStat{ - GameID: gameID, - Team: team, + GameID: gameID, Team: team, MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()), FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()), FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()), @@ -235,8 +313,7 @@ func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.Team func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat { return models.TeamGameAdvStat{ - GameID: gameID, - Team: team, + GameID: gameID, Team: team, MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()), TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()), EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()), @@ -255,8 +332,6 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa } } -// --- DB and Utility Functions --- - func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error { if len(pbs) > 0 { if err := db.Clauses(clause.OnConflict{ @@ -291,22 +366,4 @@ func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models. } } return nil -} - - - -// getModelColumns is a placeholder for a more robust reflection-based column name generator. -// For now, it returns hardcoded lists. -func getModelColumns(model interface{}) []string { - switch model.(type) { - case *models.PlayerGameBasicStat: - return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"} - case *models.PlayerGameAdvStat: - return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"} - case *models.TeamGameBasicStat: - return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"} - case *models.TeamGameAdvStat: - return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"} - } - return []string{} -} +} \ No newline at end of file diff --git a/services/helpers.go b/services/helpers.go index 96f307c..0f854a1 100644 --- a/services/helpers.go +++ b/services/helpers.go @@ -1,27 +1,49 @@ package services -import "strconv" +import ( + "log" + "strconv" + "sync" + + "gorm.io/gorm/schema" +) // mustAtoi parses s into an int, or returns 0 on error. +// This function remains unchanged for general use. func mustAtoi(s string) int { - i, _ := strconv.Atoi(s) - return i + i, _ := strconv.Atoi(s) + return i } -// mustAtoiWithSign handles strings that might have a "+" or "-" sign. +// mustAtoiWithSign is the new function to handle strings that might +// have a "+" or "-" sign, like the 'plus_minus' stat. func mustAtoiWithSign(s string) int { - if s == "" { - return 0 - } - // The strconv.Atoi function handles the sign automatically. + // strconv.Atoi already handles signs correctly. This function + // provides a clear, semantic name for its specific purpose. i, _ := strconv.Atoi(s) return i } // mustParseFloat parses s into a float64, or returns 0.0 on error. func mustParseFloat(s string) float64 { - f, _ := strconv.ParseFloat(s, 64) - return f + f, _ := strconv.ParseFloat(s, 64) + return f } +// getModelColumns uses reflection to discover model columns for dynamic upserts. +func getModelColumns(instance interface{}) []string { + s, err := schema.Parse(instance, &sync.Map{}, schema.NamingStrategy{}) + if err != nil { + log.Printf("Failed to parse GORM schema: %v", err) + return []string{} + } + columns := make([]string, 0, len(s.Fields)) + for _, field := range s.Fields { + if field.PrimaryKey { + continue // Skip primary key columns + } + columns = append(columns, field.DBName) + } + return columns +} \ No newline at end of file