From c7c04fee7ed6a13d04b15e365f0d21efe74242c4 Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Tue, 8 Jul 2025 01:23:34 -0500 Subject: [PATCH] per game stats and box score --- config/database.go | 29 ++- import.go | 51 +++- main.go | 7 + models/game.go | 184 ++++++++------- services/box_score_scrape_service.go | 334 +++++++++++++++++++++++++++ services/game_scrape_service.go | 190 +++++++++++++++ services/helpers.go | 10 + 7 files changed, 704 insertions(+), 101 deletions(-) create mode 100644 services/box_score_scrape_service.go create mode 100644 services/game_scrape_service.go diff --git a/config/database.go b/config/database.go index 2f779b3..080bf78 100644 --- a/config/database.go +++ b/config/database.go @@ -1,3 +1,5 @@ +// File: config/database.go + package config import ( @@ -12,7 +14,7 @@ import ( ) func InitDB(shouldMigrate bool) *gorm.DB { - // CHANGE: Build the DSN from environment variables + // ... (DSN setup code is unchanged) ... dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC", os.Getenv("DB_HOST"), os.Getenv("DB_USER"), @@ -21,13 +23,32 @@ func InitDB(shouldMigrate bool) *gorm.DB { os.Getenv("DB_PORT"), ) - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) // <- CHANGE: Use the postgres driver + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { log.Fatalf("failed to connect database: %v", err) } metrics.DBOperationsTotal.WithLabelValues("connect", "database").Inc() if shouldMigrate { + // --- ADD THIS BLOCK TO DROP STALE TABLES --- + // Drop tables in reverse order of dependency (children first). + // This ensures a clean migration every time the import process runs. + log.Println("⚠️ Dropping existing game-related tables for a clean migration...") + if err := db.Migrator().DropTable( + &models.LineScore{}, + &models.PlayerGameBasicStat{}, + &models.PlayerGameAdvStat{}, + &models.TeamGameBasicStat{}, + &models.TeamGameAdvStat{}, + &models.Game{}, // Drop parent table last + ); err != nil { + log.Fatalf("failed to drop tables: %v", err) + } + log.Println("✅ Tables dropped successfully.") + // --- END OF ADDED BLOCK --- + + + // Your existing AutoMigrate calls will now work correctly if err := db.AutoMigrate(&models.PlayerAdvancedStat{}); err != nil { log.Fatalf("migrate PlayerAdvancedStat: %v", err) } @@ -40,7 +61,7 @@ func InitDB(shouldMigrate bool) *gorm.DB { if err := db.AutoMigrate(&models.APIKey{}); err != nil { log.Fatalf("migrate APIKey: %v", err) } - // Game Models + // Game Models will be recreated with the correct schema if err := db.AutoMigrate( &models.Game{}, &models.LineScore{}, @@ -55,4 +76,4 @@ func InitDB(shouldMigrate bool) *gorm.DB { } return db -} +} \ No newline at end of file diff --git a/import.go b/import.go index 17043aa..4792e7d 100644 --- a/import.go +++ b/import.go @@ -11,7 +11,7 @@ import ( // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 func importPlayerAdvanced(db *gorm.DB) { - for season := 2022; season <= 2025; season++ { + for season := 2025; season <= 2025; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -23,7 +23,7 @@ func importPlayerAdvanced(db *gorm.DB) { // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 func importPlayerAdvancedPlayoffs(db *gorm.DB) { - for season := 2022; season <= 2025; season++ { + for season := 2025; season <= 2025; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -35,7 +35,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) { // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 2022; season <= 2025; season++ { + for season := 2025; season <= 2025; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } @@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 2022; season <= 2025; season++ { + for season := 2025; season <= 2025; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } @@ -57,6 +57,49 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { } } +// importGameSchedules fetches and stores game schedules +func importGameSchedules(db *gorm.DB) { + // An NBA season typically runs from October to June + months := []string{ + "october", "november", "december", "january", + "february", "march", "april", "may", "june", + } + + for season := 2025; season <= 2025; season++ { + log.Printf("--- Starting Game Schedule Import for Season: %d ---", season) + for _, month := range months { + // The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished) + if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil { + // Log the error but continue to the next month/season + log.Printf("Game schedule import failed for %s %d: %v", month, season, err) + } + log.Printf("Game schedule import for %s, %d complete.", month, season) + // Respectful delay between requests + time.Sleep(1100 * time.Millisecond) + utils.SleepWithJitter(1000 * time.Millisecond) + } + log.Printf("--- Finished Game Schedule Import for Season: %d ---", season) + } +} + + +// importBoxScores fetches and stores all box score data (line scores, player/team stats) +// for games within a recent date range. +func importBoxScores(db *gorm.DB) { + // Define the date range for the import. + // This example fetches data for games in the last 90 days. + to := time.Now() + from := to.AddDate(0, -1, 1) // 0 years, -3 months, 0 days + + log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02")) + + if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { + log.Fatalf("Box score import failed: %v", err) + } + + log.Printf("--- Finished Box Score Data Import ---") +} + diff --git a/main.go b/main.go index 49672bf..48eadbb 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,13 @@ func main() { importPlayerTotalsPlayoffsScrape(db) log.Println("🎉 Player Playoffs (scraped) Import completed successfully") + importGameSchedules(db) + log.Println("🎉 Game Imports completed successfully 🏀") + + importBoxScores(db) + log.Println("🎉 Related Box-Score Imports completed successfully 📦") + + log.Println("🏀 ALL Imports completed successfully ✅ 🙌") return } diff --git a/models/game.go b/models/game.go index b467faf..6ca8ff7 100644 --- a/models/game.go +++ b/models/game.go @@ -1,32 +1,30 @@ package models import ( - "time" - "gorm.io/gorm" + "time" ) -// Game represents the top-level information for a single NBA game, including schedule details and results. type Game struct { - ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;uniqueIndex" json:"gameId"` // A unique identifier for the game, e.g., "202504010ATL" - Date time.Time `gorm:"not null;index" json:"date"` - IsPlayoff bool `gorm:"not null;default:false;index" json:"isPlayoff"` - StartTimeET string `json:"startTimeET"` - Arena string `json:"arena"` - VisitorTeam string `gorm:"not null" json:"visitorTeam"` - VisitorPTS int `json:"visitorPts"` - HomeTeam string `gorm:"not null" json:"homeTeam"` - HomePTS int `json:"homePts"` - GameDuration string `json:"gameDuration"` // e.g., "2:23" - BoxScoreURL string `json:"boxScoreUrl"` + ID uint `gorm:"primaryKey" swaggerignore:"true"` + GameID string `gorm:"not null;uniqueIndex" json:"gameId"` // The field you want to use + Date time.Time `gorm:"not null;index" json:"date"` + IsPlayoff bool `gorm:"not null;default:false;index" json:"isPlayoff"` + StartTimeET string `json:"startTimeET"` + Arena string `json:"arena"` + VisitorTeam string `gorm:"not null" json:"visitorTeam"` + VisitorPTS int `json:"visitorPts"` + HomeTeam string `gorm:"not null" json:"homeTeam"` + HomePTS int `json:"homePts"` + GameDuration string `json:"gameDuration"` + BoxScoreURL string `json:"boxScoreUrl"` - // Associations - LineScores []LineScore `json:"lineScores"` - PlayerGameBasicStats []PlayerGameBasicStat `json:"playerGameBasicStats"` - PlayerGameAdvStats []PlayerGameAdvStat `json:"playerGameAdvStats"` - TeamGameBasicStats []TeamGameBasicStat `json:"teamGameBasicStats"` - TeamGameAdvStats []TeamGameAdvStat `json:"teamGameAdvStats"` + // Associations (Updated with explicit foreign key tags) + LineScores []LineScore `gorm:"foreignKey:GameID;references:GameID" json:"lineScores"` + PlayerGameBasicStats []PlayerGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameBasicStats"` + PlayerGameAdvStats []PlayerGameAdvStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameAdvStats"` + TeamGameBasicStats []TeamGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"teamGameBasicStats"` + TeamGameAdvStats []TeamGameAdvStat `gorm:"foreignKey:GameID;references:GameID" json:"teamGameAdvStats"` CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` @@ -43,10 +41,10 @@ type LineScore struct { Q3 int `json:"q3"` Q4 int `json:"q4"` // Overtime scores; will be 0 if the game did not go to the respective OT period. - OT1 int `json:"ot1"` - OT2 int `json:"ot2"` - OT3 int `json:"ot3"` - Total int `json:"total"` + OT1 int `json:"ot1"` + OT2 int `json:"ot2"` + OT3 int `json:"ot3"` + Total int `json:"total"` CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` @@ -55,33 +53,33 @@ type LineScore struct { // PlayerGameBasicStat contains the basic box score statistics for a single player in a single game. type PlayerGameBasicStat struct { - ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"gameId"` - PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"playerId"` - PlayerName string `json:"playerName"` - Team string `gorm:"not null" json:"team"` - Status string `json:"status"` // e.g., "Starter", "Bench", "Did Not Play" - MP string `json:"mp"` // Minutes Played, e.g., "38:07" - FG int `json:"fg"` - FGA int `json:"fga"` - FGPercent float64 `json:"fgPercent"` - ThreeP int `json:"threeP"` - ThreePA int `json:"threePa"` + ID uint `gorm:"primaryKey" swaggerignore:"true"` + GameID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"gameId"` + PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"playerId"` + PlayerName string `json:"playerName"` + Team string `gorm:"not null" json:"team"` + Status string `json:"status"` // e.g., "Starter", "Bench", "Did Not Play" + MP string `json:"mp"` // Minutes Played, e.g., "38:07" + FG int `json:"fg"` + FGA int `json:"fga"` + FGPercent float64 `json:"fgPercent"` + ThreeP int `json:"threeP"` + ThreePA int `json:"threePa"` ThreePPercent float64 `json:"threePPercent"` - FT int `json:"ft"` - FTA int `json:"fta"` - FTPercent float64 `json:"ftPercent"` - ORB int `json:"orb"` - DRB int `json:"drb"` - TRB int `json:"trb"` - AST int `json:"ast"` - STL int `json:"stl"` - BLK int `json:"blk"` - TOV int `json:"tov"` - PF int `json:"pf"` - PTS int `json:"pts"` - GmSc float64 `json:"gmSc"` // Game Score - PlusMinus int `json:"plusMinus"` + FT int `json:"ft"` + FTA int `json:"fta"` + FTPercent float64 `json:"ftPercent"` + ORB int `json:"orb"` + DRB int `json:"drb"` + TRB int `json:"trb"` + AST int `json:"ast"` + STL int `json:"stl"` + BLK int `json:"blk"` + TOV int `json:"tov"` + PF int `json:"pf"` + PTS int `json:"pts"` + GmSc float64 `json:"gmSc"` // Game Score + PlusMinus int `json:"plusMinus"` CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` @@ -90,16 +88,16 @@ type PlayerGameBasicStat struct { // PlayerGameAdvStat contains the advanced box score statistics for a single player in a single game. type PlayerGameAdvStat struct { - ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"gameId"` - PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"playerId"` - PlayerName string `json:"playerName"` - Team string `gorm:"not null" json:"team"` - MP string `json:"mp"` // Minutes Played - TSPercent float64 `json:"tsPercent"` + ID uint `gorm:"primaryKey" swaggerignore:"true"` + GameID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"gameId"` + PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"playerId"` + PlayerName string `json:"playerName"` + Team string `gorm:"not null" json:"team"` + MP string `json:"mp"` // Minutes Played + TSPercent float64 `json:"tsPercent"` EFGPercent float64 `json:"efgPercent"` - ThreePAr float64 `json:"threePAr"` - FTr float64 `json:"fTr"` + ThreePAr float64 `json:"threePAr"` + FTr float64 `json:"fTr"` ORBPercent float64 `json:"orbPercent"` DRBPercent float64 `json:"drbPercent"` TRBPercent float64 `json:"trbPercent"` @@ -108,9 +106,9 @@ type PlayerGameAdvStat struct { BLKPercent float64 `json:"blkPercent"` TOVPercent float64 `json:"tovPercent"` USGPercent float64 `json:"usgPercent"` - ORtg int `json:"oRtg"` - DRtg int `json:"dRtg"` - BPM float64 `json:"bpm"` // Box Plus/Minus + ORtg int `json:"oRtg"` + DRtg int `json:"dRtg"` + BPM float64 `json:"bpm"` // Box Plus/Minus CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` @@ -119,28 +117,28 @@ type PlayerGameAdvStat struct { // TeamGameBasicStat holds the total basic stats for a team in a single game. type TeamGameBasicStat struct { - ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"gameId"` - Team string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"team"` - MP int `json:"mp"` // Total minutes, usually 240 for a regulation game - FG int `json:"fg"` - FGA int `json:"fga"` - FGPercent float64 `json:"fgPercent"` - ThreeP int `json:"threeP"` - ThreePA int `json:"threePa"` + ID uint `gorm:"primaryKey" swaggerignore:"true"` + GameID string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"gameId"` + Team string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"team"` + MP int `json:"mp"` // Total minutes, usually 240 for a regulation game + FG int `json:"fg"` + FGA int `json:"fga"` + FGPercent float64 `json:"fgPercent"` + ThreeP int `json:"threeP"` + ThreePA int `json:"threePa"` ThreePPercent float64 `json:"threePPercent"` - FT int `json:"ft"` - FTA int `json:"fta"` - FTPercent float64 `json:"ftPercent"` - ORB int `json:"orb"` - DRB int `json:"drb"` - TRB int `json:"trb"` - AST int `json:"ast"` - STL int `json:"stl"` - BLK int `json:"blk"` - TOV int `json:"tov"` - PF int `json:"pf"` - PTS int `json:"pts"` + FT int `json:"ft"` + FTA int `json:"fta"` + FTPercent float64 `json:"ftPercent"` + ORB int `json:"orb"` + DRB int `json:"drb"` + TRB int `json:"trb"` + AST int `json:"ast"` + STL int `json:"stl"` + BLK int `json:"blk"` + TOV int `json:"tov"` + PF int `json:"pf"` + PTS int `json:"pts"` CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` @@ -149,14 +147,14 @@ type TeamGameBasicStat struct { // TeamGameAdvStat holds the total advanced stats for a team in a single game. type TeamGameAdvStat struct { - ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"gameId"` - Team string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"team"` - MP int `json:"mp"` - TSPercent float64 `json:"tsPercent"` + ID uint `gorm:"primaryKey" swaggerignore:"true"` + GameID string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"gameId"` + Team string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"team"` + MP int `json:"mp"` + TSPercent float64 `json:"tsPercent"` EFGPercent float64 `json:"efgPercent"` - ThreePAr float64 `json:"threePAr"` - FTr float64 `json:"fTr"` + ThreePAr float64 `json:"threePAr"` + FTr float64 `json:"fTr"` ORBPercent float64 `json:"orbPercent"` DRBPercent float64 `json:"drbPercent"` TRBPercent float64 `json:"trbPercent"` @@ -165,8 +163,8 @@ type TeamGameAdvStat struct { BLKPercent float64 `json:"blkPercent"` TOVPercent float64 `json:"tovPercent"` USGPercent float64 `json:"usgPercent"` // Will be ~100.0 for a team - ORtg float64 `json:"oRtg"` - DRtg float64 `json:"dRtg"` + ORtg float64 `json:"oRtg"` + DRtg float64 `json:"dRtg"` CreatedAt time.Time `swaggerignore:"true"` UpdatedAt time.Time `swaggerignore:"true"` diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go new file mode 100644 index 0000000..45932a6 --- /dev/null +++ b/services/box_score_scrape_service.go @@ -0,0 +1,334 @@ +// File: services/box_score_scrape_service.go +package services + +import ( + "fmt" + "log" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/nprasad2077/NBA_Go/models" + "github.com/nprasad2077/NBA_Go/utils" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const boxScoreURLBase = "https://www.basketball-reference.com" + +// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores. +func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error { + var games []models.Game + // Query the database for games within the specified date range. + if err := db.Where("date >= ? AND date < ?", from, to).Find(&games).Error; err != nil { + return fmt.Errorf("failed to query games from DB: %w", err) + } + + log.Printf("Found %d games to process in the specified date range.", len(games)) + + for _, game := range games { + log.Printf("Processing game: %s", game.GameID) + fullURL := boxScoreURLBase + game.BoxScoreURL + if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil { + // Log the error but continue to the next game + log.Printf("Error processing box score for game %s: %v", game.GameID, err) + } + // Be a good internet citizen and pause between requests. + utils.SleepWithJitter(1500 * time.Millisecond) + } + return nil +} + +// scrapeBoxScorePage handles fetching and parsing a single box score page. +func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("received non-200 status code: %s", resp.Status) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return err + } + + // The content is often inside a comment, so we need to extract it. + doc = uncommentDoc(doc) + + // --- Scrape all data types from the page --- + if err := parseAndStoreLineScore(db, doc, gameID); err != nil { + return err + } + if err := parseAndStoreBoxScores(db, doc, gameID); err != nil { + return err + } + + return nil +} + +// parseAndStoreLineScore scrapes the line score table. +func parseAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error { + var lineScores []models.LineScore + doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) { + ls := models.LineScore{GameID: gameID} + ls.Team = row.Find(`th[data-stat="team"] a`).Text() + ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text()) + ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text()) + ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text()) + ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text()) + ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()) + ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()) + ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()) + ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text()) + lineScores = append(lineScores, ls) + }) + + if len(lineScores) > 0 { + return db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}), + }).Create(&lineScores).Error + } + return nil +} + +// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them. +func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error { + var allPlayerBasicStats []models.PlayerGameBasicStat + var allPlayerAdvStats []models.PlayerGameAdvStat + var allTeamBasicStats []models.TeamGameBasicStat + var allTeamAdvStats []models.TeamGameAdvStat + + // Use a CSS attribute selector to find all box score tables for both teams. + doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) { + tableID, _ := table.Attr("id") + isAdvanced := strings.Contains(tableID, "-advanced") + teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic") + teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced") + + // Process player rows + table.Find("tbody tr").Each(func(j int, row *goquery.Selection) { + playerID, exists := row.Find("th").Attr("data-append-csv") + if !exists || playerID == "" { + return // Not a player row + } + + // Handle "Did Not Play" or other statuses + reason := row.Find(`td[data-stat="reason"]`) + status := "Played" + if reason.Length() > 0 { + status = reason.Text() + } + + if !isAdvanced { + stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status) + allPlayerBasicStats = append(allPlayerBasicStats, stat) + } else { + stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status) + allPlayerAdvStats = append(allPlayerAdvStats, stat) + } + }) + + // Process team total row + table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) { + if !isAdvanced { + stat := parseTeamBasicStat(row, gameID, teamAbbr) + allTeamBasicStats = append(allTeamBasicStats, stat) + } else { + stat := parseTeamAdvStat(row, gameID, teamAbbr) + allTeamAdvStats = append(allTeamAdvStats, stat) + } + }) + }) + + // Batch upsert all collected stats + if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil { + return err + } + + return nil +} + +// --- Parsing Helper Functions --- + +func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat { + return models.PlayerGameBasicStat{ + GameID: gameID, + PlayerID: playerID, + PlayerName: row.Find(`th[data-stat="player"] a`).Text(), + Team: team, + Status: status, + MP: row.Find(`td[data-stat="mp"]`).Text(), + FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()), + FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()), + FGPercent: mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()), + ThreeP: mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()), + ThreePA: mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()), + ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()), + FT: mustAtoi(row.Find(`td[data-stat="ft"]`).Text()), + FTA: mustAtoi(row.Find(`td[data-stat="fta"]`).Text()), + FTPercent: mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()), + ORB: mustAtoi(row.Find(`td[data-stat="orb"]`).Text()), + DRB: mustAtoi(row.Find(`td[data-stat="drb"]`).Text()), + TRB: mustAtoi(row.Find(`td[data-stat="trb"]`).Text()), + AST: mustAtoi(row.Find(`td[data-stat="ast"]`).Text()), + STL: mustAtoi(row.Find(`td[data-stat="stl"]`).Text()), + BLK: mustAtoi(row.Find(`td[data-stat="blk"]`).Text()), + TOV: mustAtoi(row.Find(`td[data-stat="tov"]`).Text()), + PF: mustAtoi(row.Find(`td[data-stat="pf"]`).Text()), + PTS: mustAtoi(row.Find(`td[data-stat="pts"]`).Text()), + GmSc: mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()), + PlusMinus: mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()), + } +} + +func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat { + return models.PlayerGameAdvStat{ + GameID: gameID, + PlayerID: playerID, + PlayerName: row.Find(`th[data-stat="player"] a`).Text(), + Team: team, + MP: row.Find(`td[data-stat="mp"]`).Text(), + TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()), + EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()), + ThreePAr: mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()), + FTr: mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()), + ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()), + DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()), + TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()), + ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()), + STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()), + BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()), + TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()), + USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()), + ORtg: mustAtoi(row.Find(`td[data-stat="off_rtg"]`).Text()), + DRtg: mustAtoi(row.Find(`td[data-stat="def_rtg"]`).Text()), + BPM: mustParseFloat(row.Find(`td[data-stat="bpm"]`).Text()), + } +} + +func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat { + return models.TeamGameBasicStat{ + GameID: gameID, + Team: team, + MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()), + FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()), + FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()), + FGPercent: mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()), + ThreeP: mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()), + ThreePA: mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()), + ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()), + FT: mustAtoi(row.Find(`td[data-stat="ft"]`).Text()), + FTA: mustAtoi(row.Find(`td[data-stat="fta"]`).Text()), + FTPercent: mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()), + ORB: mustAtoi(row.Find(`td[data-stat="orb"]`).Text()), + DRB: mustAtoi(row.Find(`td[data-stat="drb"]`).Text()), + TRB: mustAtoi(row.Find(`td[data-stat="trb"]`).Text()), + AST: mustAtoi(row.Find(`td[data-stat="ast"]`).Text()), + STL: mustAtoi(row.Find(`td[data-stat="stl"]`).Text()), + BLK: mustAtoi(row.Find(`td[data-stat="blk"]`).Text()), + TOV: mustAtoi(row.Find(`td[data-stat="tov"]`).Text()), + PF: mustAtoi(row.Find(`td[data-stat="pf"]`).Text()), + PTS: mustAtoi(row.Find(`td[data-stat="pts"]`).Text()), + } +} + +func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat { + return models.TeamGameAdvStat{ + GameID: gameID, + Team: team, + MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()), + TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()), + EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()), + ThreePAr: mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()), + FTr: mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()), + ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()), + DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()), + TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()), + ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()), + STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()), + BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()), + TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()), + USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()), + ORtg: mustParseFloat(row.Find(`td[data-stat="off_rtg"]`).Text()), + DRtg: mustParseFloat(row.Find(`td[data-stat="def_rtg"]`).Text()), + } +} + +// --- DB and Utility Functions --- + +func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error { + if len(pbs) > 0 { + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})), + }).Create(&pbs).Error; err != nil { + return fmt.Errorf("failed to upsert player basic stats: %w", err) + } + } + if len(pas) > 0 { + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})), + }).Create(&pas).Error; err != nil { + return fmt.Errorf("failed to upsert player advanced stats: %w", err) + } + } + if len(tbs) > 0 { + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})), + }).Create(&tbs).Error; err != nil { + return fmt.Errorf("failed to upsert team basic stats: %w", err) + } + } + if len(tas) > 0 { + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})), + }).Create(&tas).Error; err != nil { + return fmt.Errorf("failed to upsert team advanced stats: %w", err) + } + } + return nil +} + +// uncommentDoc finds and replaces commented out HTML sections. +func uncommentDoc(doc *goquery.Document) *goquery.Document { + doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) { + s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool { + return goquery.NodeName(s) == "#comment" + }).Each(func(i int, s *goquery.Selection) { + uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text())) + // FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string. + s.ReplaceWithSelection(uncommented.Find("body").Children()) + }) + }) + return doc +} + +// getModelColumns is a placeholder for a more robust reflection-based column name generator. +// For now, it returns hardcoded lists. +func getModelColumns(model interface{}) []string { + switch model.(type) { + case *models.PlayerGameBasicStat: + return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"} + case *models.PlayerGameAdvStat: + return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"} + case *models.TeamGameBasicStat: + return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"} + case *models.TeamGameAdvStat: + return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"} + } + return []string{} +} diff --git a/services/game_scrape_service.go b/services/game_scrape_service.go new file mode 100644 index 0000000..386db98 --- /dev/null +++ b/services/game_scrape_service.go @@ -0,0 +1,190 @@ +// File: services/game_scrape_service.go +package services + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/nprasad2077/NBA_Go/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const gameScheduleURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_games-%s.html" + +// FetchAndStoreGameSchedule scrapes the game schedule for a given season and month. +// The month should be the full lowercase name, e.g., "october", "november". +// If db is nil, it will perform a "dry run" and print the parsed data to the console. +func FetchAndStoreGameSchedule(db *gorm.DB, season int, month string) error { + url := fmt.Sprintf(gameScheduleURLFmt, season, month) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to fetch schedule for %s %d: %w", month, season, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("⚠️ Skipping schedule for %s %d (Status: %s)", month, season, resp.Status) + return nil // Not a fatal error, just no data for this month. + } + + htmlBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body for %s %d: %w", month, season, err) + } + + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes)) + if err != nil { + return fmt.Errorf("failed to parse HTML for %s %d: %w", month, season, err) + } + + table := doc.Find("table#schedule") + if table.Length() == 0 { + // Sometimes the content is commented out + commentNode := doc.Find("#all_schedule").Contents().FilterFunction(func(i int, s *goquery.Selection) bool { + return goquery.NodeName(s) == "#comment" + }) + if commentNode.Length() > 0 { + commentedHTML := commentNode.Nodes[0].FirstChild.Data + innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) + if err != nil { + return fmt.Errorf("failed to parse commented schedule HTML: %w", err) + } + table = innerDoc.Find("table#schedule") + } + } + + if table.Length() == 0 { + log.Printf("No schedule table found for %s %d.", month, season) + return nil + } + + var gamesToUpsert []models.Game + table.Find("tbody tr").Each(func(i int, row *goquery.Selection) { + // Skip table header rows that are sometimes repeated in the body + if row.Find("th.poptip").Length() > 1 { + return + } + + // Skip rows that don't represent games (e.g., placeholder rows) + if row.Find(`[data-stat="visitor_team_name"]`).Text() == "" { + return + } + + var game models.Game + var gameID string + + // Extract GameID from the box score link, which is the most reliable unique key + boxScoreCell := row.Find(`td[data-stat="box_score_text"] a`) + if href, exists := boxScoreCell.Attr("href"); exists { + parts := strings.Split(href, "/") + fileName := parts[len(parts)-1] + gameID = strings.TrimSuffix(fileName, ".html") + } + + // If there's no box score link, it's likely a future game, we can skip it or handle differently + if gameID == "" { + return + } + game.GameID = gameID + + // Get the date part from the 'csk' attribute for accuracy + dateCsk, _ := row.Find(`th[data-stat="date_game"]`).Attr("csk") + var datePart string + if len(dateCsk) >= 8 { + datePart = dateCsk[:8] + } else { + log.Printf("Could not parse date from invalid 'csk' attribute: %s. Skipping.", dateCsk) + return + } + + // Get the start time string + startTimeET := row.Find(`td[data-stat="game_start_time"]`).Text() + if startTimeET == "" { + // Skip games without a start time, as they can't be parsed accurately + log.Printf("Could not find start time for game %s. Skipping.", gameID) + return + } + + // Load the US/Eastern timezone to correctly handle ET/EST/EDT + eastern, err := time.LoadLocation("America/New_York") + if err != nil { + log.Printf("FATAL: Could not load America/New_York timezone: %v", err) + // This is a system-level error, so we stop the row processing here. + // The function will continue and process any games already parsed. + return + } + + // Combine date and time and parse together in the correct timezone. + // The layout "3:04p" handles times like "7:30p". + fullDateTimeString := datePart + startTimeET + layout := "200601023:04p" + gameDate, err := time.ParseInLocation(layout, fullDateTimeString, eastern) + if err != nil { + log.Printf("Could not parse combined date-time for game %s (value: '%s'): %v. Skipping.", gameID, fullDateTimeString, err) + return + } + game.Date = gameDate + game.StartTimeET = startTimeET // Keep the original string as well + + game.VisitorTeam = row.Find(`td[data-stat="visitor_team_name"] a`).Text() + game.VisitorPTS = mustAtoi(row.Find(`td[data-stat="visitor_pts"]`).Text()) + game.HomeTeam = row.Find(`td[data-stat="home_team_name"] a`).Text() + game.HomePTS = mustAtoi(row.Find(`td[data-stat="home_pts"]`).Text()) + game.BoxScoreURL, _ = boxScoreCell.Attr("href") + game.GameDuration = row.Find(`td[data-stat="game_duration"]`).Text() + game.Arena = row.Find(`td[data-stat="arena_name"]`).Text() + game.IsPlayoff = strings.Contains(row.Find(`td[data-stat="game_remarks"]`).Text(), "Playoffs") + + gamesToUpsert = append(gamesToUpsert, game) + }) + + if len(gamesToUpsert) > 0 { + // If the db connection is nil, we're in test/debug mode. Print to console. + if db == nil { + log.Println("--- RUNNING IN DRY-RUN MODE ---") + for _, game := range gamesToUpsert { + // Use %+v to print the struct with field names for clarity + log.Printf("Game Data: %+v\n", game) + } + log.Printf("--- WOULD INSERT %d RECORDS ---", len(gamesToUpsert)) + return nil // End execution for dry-run + } + + log.Printf("Attempting to batch upsert %d games for %s %d...", len(gamesToUpsert), month, season) + + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}}, + DoUpdates: clause.AssignmentColumns(allGameColumns()), + }).Create(&gamesToUpsert).Error; err != nil { + log.Printf("Failed to batch upsert games: %v", err) + return err + } + log.Printf("✅ Successfully batch upserted %d game records for %s %d.", len(gamesToUpsert), month, season) + } else { + log.Printf("No game data found to import for %s %d.", month, season) + } + + return nil +} + +// allGameColumns returns a list of all column names in the Game model for the upsert operation. +// This ensures that if a record exists, all its fields are updated with the new data. +func allGameColumns() []string { + return []string{ + "date", "is_playoff", "start_time_et", "arena", "visitor_team", + "visitor_pts", "home_team", "home_pts", "game_duration", "box_score_url", + "updated_at", + } +} diff --git a/services/helpers.go b/services/helpers.go index 324676e..96f307c 100644 --- a/services/helpers.go +++ b/services/helpers.go @@ -8,6 +8,16 @@ func mustAtoi(s string) int { return i } +// mustAtoiWithSign handles strings that might have a "+" or "-" sign. +func mustAtoiWithSign(s string) int { + if s == "" { + return 0 + } + // The strconv.Atoi function handles the sign automatically. + i, _ := strconv.Atoi(s) + return i +} + // mustParseFloat parses s into a float64, or returns 0.0 on error. func mustParseFloat(s string) float64 { f, _ := strconv.ParseFloat(s, 64)