From e6851b3bb619313f2b95d54e36d74bafae0ad99f Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Thu, 19 Jun 2025 22:20:43 -0500 Subject: [PATCH] batch insert adv --- import.go | 48 +-- main.go | 8 +- ...player_advanced_playoffs_scrape_service.go | 359 +++++++++--------- 3 files changed, 211 insertions(+), 204 deletions(-) diff --git a/import.go b/import.go index 3ab6a1f..1aa3332 100644 --- a/import.go +++ b/import.go @@ -9,33 +9,33 @@ import ( "github.com/nprasad2077/NBA_Go/utils" ) -// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 -// func importPlayerAdvanced(db *gorm.DB) { -// for season := 2013; season <= 2014; season++ { -// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { -// log.Printf("advanced import failed for %d: %v", season, err) -// } -// log.Printf("Advanced import for season: %d", season) -// time.Sleep(1100 * time.Millisecond) -// utils.SleepWithJitter(1000 * time.Millisecond) -// } -// } +// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 +func importPlayerAdvanced(db *gorm.DB) { + for season := 2000; season <= 2014; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced import for season: %d", season) + time.Sleep(1100 * time.Millisecond) + utils.SleepWithJitter(1000 * time.Millisecond) + } +} -// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 -// func importPlayerAdvancedPlayoffs(db *gorm.DB) { -// for season := 2013; season <= 2014; season++ { -// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { -// log.Printf("advanced import failed for %d: %v", season, err) -// } -// log.Printf("Advanced Playoffs import for season: %d", season) -// time.Sleep(1100 * time.Millisecond) -// utils.SleepWithJitter(1250 * time.Millisecond) -// } -// } +// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 +func importPlayerAdvancedPlayoffs(db *gorm.DB) { + for season := 2000; season <= 2014; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced Playoffs import for season: %d", season) + time.Sleep(1100 * time.Millisecond) + utils.SleepWithJitter(1250 * time.Millisecond) + } +} // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 2013; season <= 2014; season++ { + for season := 2000; season <= 2012; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } @@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 2013; season <= 2014; season++ { + for season := 2000; season <= 2012; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } diff --git a/main.go b/main.go index 64f31bf..0b66512 100644 --- a/main.go +++ b/main.go @@ -45,11 +45,11 @@ func main() { // Run all migrations + import steps exactly once db := config.InitDB(true) - // importPlayerAdvanced(db) - // log.Println("🎉 Player Advanced Import completed successfully") + importPlayerAdvanced(db) + log.Println("🎉 Player Advanced Import completed successfully") - // importPlayerAdvancedPlayoffs(db) - // log.Println("🎉 Player Advanced Playoffs Import completed successfully") + importPlayerAdvancedPlayoffs(db) + log.Println("🎉 Player Advanced Playoffs Import completed successfully") importPlayerTotalsScrape(db) log.Println("🎉 Player Totals (scraped) Import completed successfully") diff --git a/services/player_advanced_playoffs_scrape_service.go b/services/player_advanced_playoffs_scrape_service.go index 9f19007..8a22eb3 100644 --- a/services/player_advanced_playoffs_scrape_service.go +++ b/services/player_advanced_playoffs_scrape_service.go @@ -3,207 +3,214 @@ package services import ( - "bytes" - "fmt" - "io" - "log" - "net/http" - "strings" + "bytes" + "fmt" + "io" + "log" + "net/http" + "strings" - "github.com/PuerkitoBio/goquery" - "github.com/nprasad2077/NBA_Go/models" - "gorm.io/gorm" - "gorm.io/gorm/clause" + "github.com/PuerkitoBio/goquery" + "github.com/nprasad2077/NBA_Go/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" ) const ( - advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html" - advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html" + advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html" + advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html" ) // urlForAdvSeason picks the correct URL based on isPlayoff. func urlForAdvSeason(season int, isPlayoff bool) string { - if isPlayoff { - return fmt.Sprintf(advancedPlayoffURLFmt, season) - } - return fmt.Sprintf(advancedURLFmt, season) + if isPlayoff { + return fmt.Sprintf(advancedPlayoffURLFmt, season) + } + return fmt.Sprintf(advancedURLFmt, season) } -// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table -// (regular or playoffs) and upserts into the PlayerAdvancedStat model. +// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs) +// and batch upserts the data into the PlayerAdvancedStat model. func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error { - url := urlForAdvSeason(season, isPlayoff) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + url := urlForAdvSeason(season, isPlayoff) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - htmlBytes, err := io.ReadAll(resp.Body) - if err != nil { - return err - } + htmlBytes, err := io.ReadAll(resp.Body) + if err != nil { + return err + } - doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes)) - if err != nil { - return err - } + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes)) + if err != nil { + return err + } - // 1) Determine parent wrapper and table selector - // - Regular season:
- // - Playoffs:
- var parentDivSelector, tableSelector string - if isPlayoff { - parentDivSelector = "#all_advanced_stats" - tableSelector = "table#advanced_stats" - } else { - parentDivSelector = "#all_advanced" - tableSelector = "table#advanced" - } + // 1) Determine parent wrapper and table selector. This logic is complex because + // the table is often hidden inside an HTML comment. + var parentDivSelector, tableSelector string + if isPlayoff { + parentDivSelector = "#all_advanced_stats" + tableSelector = "table#advanced_stats" + } else { + parentDivSelector = "#all_advanced" + tableSelector = "table#advanced" + } - // Attempt to find the table node directly (it will be inside a comment for regular season) - table := doc.Find(parentDivSelector + " " + tableSelector) - if table.Length() == 0 { - // Look for the commented‐out HTML inside parentDivSelector - commentSel := doc. - Find(parentDivSelector). - Contents(). - FilterFunction(func(i int, s *goquery.Selection) bool { - return goquery.NodeName(s) == "#comment" - }) + table := doc.Find(parentDivSelector + " " + tableSelector) + if table.Length() == 0 { + commentSel := doc. + Find(parentDivSelector). + Contents(). + FilterFunction(func(i int, s *goquery.Selection) bool { + return goquery.NodeName(s) == "#comment" + }) - if commentSel.Length() == 0 { - return fmt.Errorf("could not find advanced stats table (even inside comment)") - } + if commentSel.Length() == 0 { + return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season) + } - // Extract the raw HTML string from the comment, then re‐parse - commentedHTML := commentSel.Nodes[0].FirstChild.Data - innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) - if err != nil { - return fmt.Errorf("failed to parse commented advanced HTML: %w", err) - } - table = innerDoc.Find(tableSelector) - if table.Length() == 0 { - return fmt.Errorf("could not find advanced stats table after un‐commenting") - } - } + commentedHTML := commentSel.Nodes[0].FirstChild.Data + innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) + if err != nil { + return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err) + } + table = innerDoc.Find(tableSelector) + if table.Length() == 0 { + return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season) + } + } - // 2) Collect the data-stat keys in header order - var headers []string - table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { - if stat, ok := th.Attr("data-stat"); ok && stat != "" { - headers = append(headers, stat) - } - }) - // Add our appended‐player column - headers = append(headers, "player-additional") + // 2) Collect the data-stat keys in header order. + var headers []string + table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { + if stat, ok := th.Attr("data-stat"); ok && stat != "" { + headers = append(headers, stat) + } + }) + headers = append(headers, "player-additional") - // 3) Iterate each row - table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { - if tr.HasClass("thead") { - return // skip repeated header rows - } - cells := tr.Find("th, td") - data := make(map[string]string, len(headers)) - var playerID string + // --- BATCHING LOGIC START --- + // Create a slice to hold all the player stats parsed from the page. + var statsToUpsert []models.PlayerAdvancedStat - cells.Each(func(i int, cell *goquery.Selection) { - key := headers[i] - data[key] = strings.TrimSpace(cell.Text()) - if id, ok := cell.Attr("data-append-csv"); ok { - playerID = id - } - }) - if playerID == "" { - // not a real data row - return - } - data["player-additional"] = playerID + // 3) Iterate each row and collect player stats into the slice. + table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { + if tr.HasClass("thead") { + return // skip repeated header rows + } + cells := tr.Find("th, td") + data := make(map[string]string, len(headers)) + var playerID string - // 4) Determine ExternalID, PlayerName, Team - // • Playoff pages use "rk", "player", "team_id" - // • Regular‐season advanced uses "ranker", "name_display", "team_name_abbr" - extID := mustAtoi(data["rk"]) - if extID == 0 { - extID = mustAtoi(data["ranker"]) - } + cells.Each(func(i int, cell *goquery.Selection) { + key := headers[i] + data[key] = strings.TrimSpace(cell.Text()) + if id, ok := cell.Attr("data-append-csv"); ok { + playerID = id + } + }) + if playerID == "" { + return // not a real data row + } + data["player-additional"] = playerID - playerName := data["player"] - if playerName == "" { - playerName = data["name_display"] - } + // 4) Determine ExternalID, PlayerName, Team. + extID := mustAtoi(data["rk"]) + if extID == 0 { + extID = mustAtoi(data["ranker"]) + } - teamID := data["team_id"] - if teamID == "" { - teamID = data["team_name_abbr"] - } + playerName := data["player"] + if playerName == "" { + playerName = data["name_display"] + } - // 5) Games column is always "g" in advanced (playoffs or season) - g := mustAtoi(data["games"]) - if g == 0 { - g = mustAtoi(data["g"]) - } + teamID := data["team_id"] + if teamID == "" { + teamID = data["team_name_abbr"] + } - // 6) Map into your GORM model - stat := models.PlayerAdvancedStat{ - ExternalID: extID, - PlayerID: playerID, - PlayerName: playerName, - Position: data["pos"], - Age: mustAtoi(data["age"]), - Games: g, - MinutesPlayed: mustAtoi(data["mp"]), - PER: mustParseFloat(data["per"]), - TSPercent: mustParseFloat(data["ts_pct"]), - ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]), - FTR: mustParseFloat(data["fta_per_fga_pct"]), - OffensiveRBPercent: mustParseFloat(data["orb_pct"]), - DefensiveRBPercent: mustParseFloat(data["drb_pct"]), - TotalRBPercent: mustParseFloat(data["trb_pct"]), - AssistPercent: mustParseFloat(data["ast_pct"]), - StealPercent: mustParseFloat(data["stl_pct"]), - BlockPercent: mustParseFloat(data["blk_pct"]), - TurnoverPercent: mustParseFloat(data["tov_pct"]), - UsagePercent: mustParseFloat(data["usg_pct"]), - OffensiveWS: mustParseFloat(data["ows"]), - DefensiveWS: mustParseFloat(data["dws"]), - WinShares: mustParseFloat(data["ws"]), - WinSharesPer: mustParseFloat(data["ws_per_48"]), - OffensiveBox: mustParseFloat(data["obpm"]), - DefensiveBox: mustParseFloat(data["dbpm"]), - Box: mustParseFloat(data["bpm"]), - VORP: mustParseFloat(data["vorp"]), - Team: teamID, - Season: season, - IsPlayoff: isPlayoff, - } + // 5) Games column is always "g" in advanced tables. + g := mustAtoi(data["games"]) + if g == 0 { + g = mustAtoi(data["g"]) + } - // 7) Upsert on (player_id, season, team, is_playoff) - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{ - {Name: "player_id"}, - {Name: "season"}, - {Name: "team"}, - {Name: "is_playoff"}, - }, - DoUpdates: clause.AssignmentColumns([]string{ - "external_id", "player_name", "position", "age", "games", - "minutes_played", - "per", "ts_percent", "three_par", "ftr", // ← use three_par & ftr - "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent", - "assist_percent", "steal_percent", "block_percent", "turnover_percent", - "usage_percent", "offensive_ws", "defensive_ws", "win_shares", - "win_shares_per", "offensive_box", "defensive_box", "box", "vorp", - }), - }).Create(&stat).Error; err != nil { - log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err) - } - }) + // 6) Map into your GORM model. + stat := models.PlayerAdvancedStat{ + ExternalID: extID, + PlayerID: playerID, + PlayerName: playerName, + Position: data["pos"], + Age: mustAtoi(data["age"]), + Games: g, + MinutesPlayed: mustAtoi(data["mp"]), + PER: mustParseFloat(data["per"]), + TSPercent: mustParseFloat(data["ts_pct"]), + ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]), + FTR: mustParseFloat(data["fta_per_fga_pct"]), + OffensiveRBPercent: mustParseFloat(data["orb_pct"]), + DefensiveRBPercent: mustParseFloat(data["drb_pct"]), + TotalRBPercent: mustParseFloat(data["trb_pct"]), + AssistPercent: mustParseFloat(data["ast_pct"]), + StealPercent: mustParseFloat(data["stl_pct"]), + BlockPercent: mustParseFloat(data["blk_pct"]), + TurnoverPercent: mustParseFloat(data["tov_pct"]), + UsagePercent: mustParseFloat(data["usg_pct"]), + OffensiveWS: mustParseFloat(data["ows"]), + DefensiveWS: mustParseFloat(data["dws"]), + WinShares: mustParseFloat(data["ws"]), + WinSharesPer: mustParseFloat(data["ws_per_48"]), + OffensiveBox: mustParseFloat(data["obpm"]), + DefensiveBox: mustParseFloat(data["dbpm"]), + Box: mustParseFloat(data["bpm"]), + VORP: mustParseFloat(data["vorp"]), + Team: teamID, + Season: season, + IsPlayoff: isPlayoff, + } - return nil + // Add the parsed stat object to our slice. + statsToUpsert = append(statsToUpsert, stat) + }) + + // 7) Perform the batch upsert operation after collecting all rows. + if len(statsToUpsert) > 0 { + log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season) + + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "player_id"}, + {Name: "season"}, + {Name: "team"}, + {Name: "is_playoff"}, + }, + DoUpdates: clause.AssignmentColumns([]string{ + "external_id", "player_name", "position", "age", "games", + "minutes_played", "per", "ts_percent", "three_par", "ftr", + "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent", + "assist_percent", "steal_percent", "block_percent", "turnover_percent", + "usage_percent", "offensive_ws", "defensive_ws", "win_shares", + "win_shares_per", "offensive_box", "defensive_box", "box", "vorp", + }), + }).Create(&statsToUpsert).Error; err != nil { + log.Printf("Failed to batch upsert advanced player stats: %v", err) + return err + } + log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season) + } else { + log.Printf("No advanced player data found to import for season %d.", season) + } + // --- BATCHING LOGIC END --- + + return nil } \ No newline at end of file