diff --git a/import.go b/import.go index bfe2a4b..3ab6a1f 100644 --- a/import.go +++ b/import.go @@ -9,33 +9,33 @@ import ( "github.com/nprasad2077/NBA_Go/utils" ) -// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 -func importPlayerAdvanced(db *gorm.DB) { - for season := 2015; season <= 2019; season++ { - if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { - log.Printf("advanced import failed for %d: %v", season, err) - } - log.Printf("Advanced import for season: %d", season) - time.Sleep(1100 * time.Millisecond) - utils.SleepWithJitter(1000 * time.Millisecond) - } -} +// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 +// func importPlayerAdvanced(db *gorm.DB) { +// for season := 2013; season <= 2014; season++ { +// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { +// log.Printf("advanced import failed for %d: %v", season, err) +// } +// log.Printf("Advanced import for season: %d", season) +// time.Sleep(1100 * time.Millisecond) +// utils.SleepWithJitter(1000 * time.Millisecond) +// } +// } -// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 -func importPlayerAdvancedPlayoffs(db *gorm.DB) { - for season := 2015; season <= 2019; season++ { - if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { - log.Printf("advanced import failed for %d: %v", season, err) - } - log.Printf("Advanced Playoffs import for season: %d", season) - time.Sleep(1100 * time.Millisecond) - utils.SleepWithJitter(1250 * time.Millisecond) - } -} +// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 +// func importPlayerAdvancedPlayoffs(db *gorm.DB) { +// for season := 2013; season <= 2014; season++ { +// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { +// log.Printf("advanced import failed for %d: %v", season, err) +// } +// log.Printf("Advanced Playoffs import for season: %d", season) +// time.Sleep(1100 * time.Millisecond) +// utils.SleepWithJitter(1250 * time.Millisecond) +// } +// } // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 2015; season <= 2019; season++ { + for season := 2013; season <= 2014; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } @@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 2015; season <= 2019; season++ { + for season := 2013; season <= 2014; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } diff --git a/main.go b/main.go index 0b66512..64f31bf 100644 --- a/main.go +++ b/main.go @@ -45,11 +45,11 @@ func main() { // Run all migrations + import steps exactly once db := config.InitDB(true) - importPlayerAdvanced(db) - log.Println("🎉 Player Advanced Import completed successfully") + // importPlayerAdvanced(db) + // log.Println("🎉 Player Advanced Import completed successfully") - importPlayerAdvancedPlayoffs(db) - log.Println("🎉 Player Advanced Playoffs Import completed successfully") + // importPlayerAdvancedPlayoffs(db) + // log.Println("🎉 Player Advanced Playoffs Import completed successfully") importPlayerTotalsScrape(db) log.Println("🎉 Player Totals (scraped) Import completed successfully") diff --git a/services/player_total_scrape_service.go b/services/player_total_scrape_service.go index 7006317..45c2a78 100644 --- a/services/player_total_scrape_service.go +++ b/services/player_total_scrape_service.go @@ -3,209 +3,205 @@ package services import ( - "bytes" - "fmt" - "io" - "log" - "net/http" - "strings" + "bytes" + "fmt" + "io" + "log" + "net/http" + "strings" - "github.com/PuerkitoBio/goquery" - "github.com/nprasad2077/NBA_Go/models" - "gorm.io/gorm" - "gorm.io/gorm/clause" + "github.com/PuerkitoBio/goquery" + "github.com/nprasad2077/NBA_Go/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" ) const ( - regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html" - playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html" + regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html" + playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html" ) // urlForSeason chooses regular vs. playoff URL. func urlForSeason(season int, isPlayoff bool) string { - if isPlayoff { - return fmt.Sprintf(playoffURLFmt, season) - } - return fmt.Sprintf(regularURLFmt, season) + if isPlayoff { + return fmt.Sprintf(playoffURLFmt, season) + } + return fmt.Sprintf(regularURLFmt, season) } // FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs) -// and upserts into PlayerTotalStat. +// and batch upserts them into PlayerTotalStat for significantly better performance. func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error { - url := urlForSeason(season, isPlayoff) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + url := urlForSeason(season, isPlayoff) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } - doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) - if err != nil { - return err - } + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) + if err != nil { + return err + } - table := doc.Find("table#totals_stats") - if table.Length() == 0 { - return fmt.Errorf("could not find table#totals_stats") - } + table := doc.Find("table#totals_stats") + if table.Length() == 0 { + return fmt.Errorf("could not find table#totals_stats for season %d", season) + } - // table := doc.Find("table#totals_stats") - // if table.Length() == 0 { - // commentSel := doc. - // Find("div#all_totals_stats"). - // Contents(). - // FilterFunction(func(i int, s *goquery.Selection) bool { - // return goquery.NodeName(s) == "#comment" - // }) + // 1) Collect the data-stat keys in header order. + var headers []string + table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { + if stat, ok := th.Attr("data-stat"); ok && stat != "" { + headers = append(headers, stat) + } + }) + headers = append(headers, "player-additional") // Add the appended-player column. - // if commentSel.Length() == 0 { - // return fmt.Errorf("could not find table#totals_stats (even inside comment)") - // } + // --- BATCHING LOGIC START --- + // Create a slice to hold all the player stats parsed from the page. + var statsToUpsert []models.PlayerTotalStat - // commentedHTML := commentSel.Nodes[0].FirstChild.Data - // innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) - // if err != nil { - // return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err) - // } - // table = innerDoc.Find("table#totals_stats") - // if table.Length() == 0 { - // return fmt.Errorf("could not find table#totals_stats after un‐commenting") - // } - // } + // 2) Iterate rows and collect all player stats into the slice. + table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { + if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") { + return // Skip repeated header rows inside the table body. + } - // 1) collect the data-stat keys in header order - var headers []string - table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { - if stat, ok := th.Attr("data-stat"); ok && stat != "" { - headers = append(headers, stat) - } - }) - // add the appended-player column - headers = append(headers, "player-additional") + cells := tr.Find("th, td") + data := make(map[string]string, len(headers)) + var playerID string - // 2) iterate rows - table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { - if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") { - return // skip header rows - } + cells.Each(func(i int, cell *goquery.Selection) { + text := strings.TrimSpace(cell.Text()) + key := headers[i] + data[key] = text + if id, ok := cell.Attr("data-append-csv"); ok { + playerID = id + } + }) - cells := tr.Find("th, td") - data := make(map[string]string, len(headers)) - var playerID string + // Skip rows that aren't actual player data rows. + if playerID == "" { + return + } + data["player-additional"] = playerID - cells.Each(func(i int, cell *goquery.Selection) { - text := strings.TrimSpace(cell.Text()) - key := headers[i] - data[key] = text - if id, ok := cell.Attr("data-append-csv"); ok { - playerID = id - } - }) - if playerID == "" { - return - } - data["player-additional"] = playerID + // 3a) Pick the right “ExternalID” key. + extID := mustAtoi(data["rk"]) + if extID == 0 { + extID = mustAtoi(data["ranker"]) + } - // 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”) - extID := mustAtoi(data["rk"]) - if extID == 0 { - extID = mustAtoi(data["ranker"]) - } + // 3b) Pick the right “PlayerName” key. + playerName := data["player"] + if playerName == "" { + playerName = data["name_display"] + } - // 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”) - playerName := data["player"] - if playerName == "" { - playerName = data["name_display"] - } + // 3c) Pick the right “Team” key. + teamID := data["team_id"] + if teamID == "" { + teamID = data["team_name_abbr"] + } - // 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”) - teamID := data["team_id"] - if teamID == "" { - teamID = data["team_name_abbr"] - } + // Pick “games”. + g := mustAtoi(data["games"]) + if g == 0 { + g = mustAtoi(data["g"]) + } - // pick “games” → fallback to “g” if empty - g := mustAtoi(data["games"]) - if g == 0 { - g = mustAtoi(data["g"]) - } + // Pick “games_started”. + gs := mustAtoi(data["games_started"]) + if gs == 0 { + gs = mustAtoi(data["gs"]) + } - // pick “games_started” → fallback to “gs” if empty - gs := mustAtoi(data["games_started"]) - if gs == 0 { - gs = mustAtoi(data["gs"]) - } + stat := models.PlayerTotalStat{ + ExternalID: extID, + PlayerID: playerID, + PlayerName: playerName, + Position: data["pos"], + Age: mustAtoi(data["age"]), + Games: g, + GamesStarted: gs, + MinutesPG: mustParseFloat(data["mp"]), + FieldGoals: mustAtoi(data["fg"]), + FieldAttempts: mustAtoi(data["fga"]), + FieldPercent: mustParseFloat(data["fg_pct"]), + ThreeFG: mustAtoi(data["fg3"]), + ThreeAttempts: mustAtoi(data["fg3a"]), + ThreePercent: mustParseFloat(data["fg3_pct"]), + TwoFG: mustAtoi(data["fg2"]), + TwoAttempts: mustAtoi(data["fg2a"]), + TwoPercent: mustParseFloat(data["fg2_pct"]), + EffectFGPercent: mustParseFloat(data["efg_pct"]), + FT: mustAtoi(data["ft"]), + FTAttempts: mustAtoi(data["fta"]), + FTPercent: mustParseFloat(data["ft_pct"]), + OffensiveRB: mustAtoi(data["orb"]), + DefensiveRB: mustAtoi(data["drb"]), + TotalRB: mustAtoi(data["trb"]), + Assists: mustAtoi(data["ast"]), + Steals: mustAtoi(data["stl"]), + Blocks: mustAtoi(data["blk"]), + Turnovers: mustAtoi(data["tov"]), + PersonalFouls: mustAtoi(data["pf"]), + Points: mustAtoi(data["pts"]), + Team: teamID, + Season: season, + IsPlayoff: isPlayoff, + } - stat := models.PlayerTotalStat{ - ExternalID: extID, - PlayerID: playerID, - PlayerName: playerName, - Position: data["pos"], - Age: mustAtoi(data["age"]), - Games: g, - GamesStarted: gs, - MinutesPG: mustParseFloat(data["mp"]), - FieldGoals: mustAtoi(data["fg"]), - FieldAttempts: mustAtoi(data["fga"]), - FieldPercent: mustParseFloat(data["fg_pct"]), - ThreeFG: mustAtoi(data["fg3"]), - ThreeAttempts: mustAtoi(data["fg3a"]), - ThreePercent: mustParseFloat(data["fg3_pct"]), - TwoFG: mustAtoi(data["fg2"]), - TwoAttempts: mustAtoi(data["fg2a"]), - TwoPercent: mustParseFloat(data["fg2_pct"]), - EffectFGPercent: mustParseFloat(data["efg_pct"]), - FT: mustAtoi(data["ft"]), - FTAttempts: mustAtoi(data["fta"]), - FTPercent: mustParseFloat(data["ft_pct"]), - OffensiveRB: mustAtoi(data["orb"]), - DefensiveRB: mustAtoi(data["drb"]), - TotalRB: mustAtoi(data["trb"]), - Assists: mustAtoi(data["ast"]), - Steals: mustAtoi(data["stl"]), - Blocks: mustAtoi(data["blk"]), - Turnovers: mustAtoi(data["tov"]), - PersonalFouls: mustAtoi(data["pf"]), - Points: mustAtoi(data["pts"]), - Team: teamID, - Season: season, - IsPlayoff: isPlayoff, - } + // Add the parsed stat object to our slice instead of writing to the DB immediately. + statsToUpsert = append(statsToUpsert, stat) + }) - // 4) upsert on (player_id, season, team, is_playoff) - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{ - {Name: "player_id"}, - {Name: "season"}, - {Name: "team"}, - {Name: "is_playoff"}, - }, - DoUpdates: clause.AssignmentColumns([]string{ - "external_id", "player_name", "position", "age", - "games", "games_started", "minutes_pg", - "field_goals", "field_attempts", "field_percent", - "three_fg", "three_attempts", "three_percent", - "two_fg", "two_attempts", "two_percent", - "effect_fg_percent", - "ft", "ft_attempts", "ft_percent", - "offensive_rb", "defensive_rb", "total_rb", - "assists", "steals", "blocks", "turnovers", - "personal_fouls", "points", - }), - }).Create(&stat).Error; err != nil { - log.Printf("Failed upsert for %s: %v", stat.PlayerID, err) - } - }) + // 3) Perform the batch upsert operation after collecting all rows. + if len(statsToUpsert) > 0 { + log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season) - return nil + // GORM's OnConflict clause works with slices, performing the batch operation efficiently. + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "player_id"}, + {Name: "season"}, + {Name: "team"}, + {Name: "is_playoff"}, + }, + DoUpdates: clause.AssignmentColumns([]string{ + "external_id", "player_name", "position", "age", + "games", "games_started", "minutes_pg", + "field_goals", "field_attempts", "field_percent", + "three_fg", "three_attempts", "three_percent", + "two_fg", "two_attempts", "two_percent", + "effect_fg_percent", + "ft", "ft_attempts", "ft_percent", + "offensive_rb", "defensive_rb", "total_rb", + "assists", "steals", "blocks", "turnovers", + "personal_fouls", "points", + }), + }).Create(&statsToUpsert).Error; err != nil { + // If the batch operation fails, log the error and return it. + log.Printf("Failed to batch upsert player total stats: %v", err) + return err + } + + log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season) + } else { + log.Printf("No player data found to import for season %d.", season) + } + // --- BATCHING LOGIC END --- + + return nil } \ No newline at end of file