nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

batch insert adv

Browse Source
This commit is contained in:
Ravi Prasad
2025-06-19 22:20:43 -05:00
parent 1a2a4c1b73
commit e6851b3bb6
3 changed files with 211 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
import.go
+24 -24
View File
@@ -9,33 +9,33 @@ import (
"github.com/nprasad2077/NBA_Go/utils" "github.com/nprasad2077/NBA_Go/utils"
) )
// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
// func importPlayerAdvanced(db *gorm.DB) { func importPlayerAdvanced(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ { for season := 2000; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err) log.Printf("advanced import failed for %d: %v", season, err)
// } }
// log.Printf("Advanced import for season: %d", season) log.Printf("Advanced import for season: %d", season)
// time.Sleep(1100 * time.Millisecond) time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1000 * time.Millisecond) utils.SleepWithJitter(1000 * time.Millisecond)
// } }
// } }
// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
// func importPlayerAdvancedPlayoffs(db *gorm.DB) { func importPlayerAdvancedPlayoffs(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ { for season := 2000; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err) log.Printf("advanced import failed for %d: %v", season, err)
// } }
// log.Printf("Advanced Playoffs import for season: %d", season) log.Printf("Advanced Playoffs import for season: %d", season)
// time.Sleep(1100 * time.Millisecond) time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1250 * time.Millisecond) utils.SleepWithJitter(1250 * time.Millisecond)
// } }
// } }
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) { func importPlayerTotalsScrape(db *gorm.DB) {
for season := 2013; season <= 2014; season++ { for season := 2000; season <= 2012; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err) log.Printf("scraped totals import failed for %d: %v", season, err)
} }
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2013; season <= 2014; season++ { for season := 2000; season <= 2012; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err) log.Printf("scraped playoffs import failed for %d: %v", season, err)
} }
main.go
+4 -4
View File
@@ -45,11 +45,11 @@ func main() {
// Run all migrations + import steps exactly once // Run all migrations + import steps exactly once
db := config.InitDB(true) db := config.InitDB(true)
// importPlayerAdvanced(db) importPlayerAdvanced(db)
// log.Println("🎉 Player Advanced Import completed successfully") log.Println("🎉 Player Advanced Import completed successfully")
// importPlayerAdvancedPlayoffs(db) importPlayerAdvancedPlayoffs(db)
// log.Println("🎉 Player Advanced Playoffs Import completed successfully") log.Println("🎉 Player Advanced Playoffs Import completed successfully")
importPlayerTotalsScrape(db) importPlayerTotalsScrape(db)
log.Println("🎉 Player Totals (scraped) Import completed successfully") log.Println("🎉 Player Totals (scraped) Import completed successfully")
services/player_advanced_playoffs_scrape_service.go
+183 -176
View File
@@ -3,207 +3,214 @@
package services package services
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"io" "io"
"log" "log"
"net/http" "net/http"
"strings" "strings"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/nprasad2077/NBA_Go/models" "github.com/nprasad2077/NBA_Go/models"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )
const ( const (
advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html" advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html" advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
) )
// urlForAdvSeason picks the correct URL based on isPlayoff. // urlForAdvSeason picks the correct URL based on isPlayoff.
func urlForAdvSeason(season int, isPlayoff bool) string { func urlForAdvSeason(season int, isPlayoff bool) string {
if isPlayoff { if isPlayoff {
return fmt.Sprintf(advancedPlayoffURLFmt, season) return fmt.Sprintf(advancedPlayoffURLFmt, season)
} }
return fmt.Sprintf(advancedURLFmt, season) return fmt.Sprintf(advancedURLFmt, season)
} }
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table // FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs)
// (regular or playoffs) and upserts into the PlayerAdvancedStat model. // and batch upserts the data into the PlayerAdvancedStat model.
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error { func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
url := urlForAdvSeason(season, isPlayoff) url := urlForAdvSeason(season, isPlayoff)
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return err return err
} }
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)") req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
resp, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return err return err
} }
defer resp.Body.Close() defer resp.Body.Close()
htmlBytes, err := io.ReadAll(resp.Body) htmlBytes, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return err return err
} }
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes)) doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
if err != nil { if err != nil {
return err return err
} }
// 1) Determine parent wrapper and table selector // 1) Determine parent wrapper and table selector. This logic is complex because
// - Regular season: <div id="all_advanced"><!-- <table id="advanced">…</table> --></div> // the table is often hidden inside an HTML comment.
// - Playoffs: <div id="all_advanced_stats"><!-- <table id="advanced_stats">…</table> --></div> var parentDivSelector, tableSelector string
var parentDivSelector, tableSelector string if isPlayoff {
if isPlayoff { parentDivSelector = "#all_advanced_stats"
parentDivSelector = "#all_advanced_stats" tableSelector = "table#advanced_stats"
tableSelector = "table#advanced_stats" } else {
} else { parentDivSelector = "#all_advanced"
parentDivSelector = "#all_advanced" tableSelector = "table#advanced"
tableSelector = "table#advanced" }
}
// Attempt to find the table node directly (it will be inside a comment for regular season) table := doc.Find(parentDivSelector + " " + tableSelector)
table := doc.Find(parentDivSelector + " " + tableSelector) if table.Length() == 0 {
if table.Length() == 0 { commentSel := doc.
// Look for the commented‐out HTML inside parentDivSelector Find(parentDivSelector).
commentSel := doc. Contents().
Find(parentDivSelector). FilterFunction(func(i int, s *goquery.Selection) bool {
Contents(). return goquery.NodeName(s) == "#comment"
FilterFunction(func(i int, s *goquery.Selection) bool { })
return goquery.NodeName(s) == "#comment"
})
if commentSel.Length() == 0 { if commentSel.Length() == 0 {
return fmt.Errorf("could not find advanced stats table (even inside comment)") return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season)
} }
// Extract the raw HTML string from the comment, then re‐parse commentedHTML := commentSel.Nodes[0].FirstChild.Data
commentedHTML := commentSel.Nodes[0].FirstChild.Data innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) if err != nil {
if err != nil { return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err)
return fmt.Errorf("failed to parse commented advanced HTML: %w", err) }
} table = innerDoc.Find(tableSelector)
table = innerDoc.Find(tableSelector) if table.Length() == 0 {
if table.Length() == 0 { return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season)
return fmt.Errorf("could not find advanced stats table after un‐commenting") }
} }
}
// 2) Collect the data-stat keys in header order // 2) Collect the data-stat keys in header order.
var headers []string var headers []string
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
if stat, ok := th.Attr("data-stat"); ok && stat != "" { if stat, ok := th.Attr("data-stat"); ok && stat != "" {
headers = append(headers, stat) headers = append(headers, stat)
} }
}) })
// Add our appended‐player column headers = append(headers, "player-additional")
headers = append(headers, "player-additional")
// 3) Iterate each row // --- BATCHING LOGIC START ---
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { // Create a slice to hold all the player stats parsed from the page.
if tr.HasClass("thead") { var statsToUpsert []models.PlayerAdvancedStat
return // skip repeated header rows
}
cells := tr.Find("th, td")
data := make(map[string]string, len(headers))
var playerID string
cells.Each(func(i int, cell *goquery.Selection) { // 3) Iterate each row and collect player stats into the slice.
key := headers[i] table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
data[key] = strings.TrimSpace(cell.Text()) if tr.HasClass("thead") {
if id, ok := cell.Attr("data-append-csv"); ok { return // skip repeated header rows
playerID = id }
} cells := tr.Find("th, td")
}) data := make(map[string]string, len(headers))
if playerID == "" { var playerID string
// not a real data row
return
}
data["player-additional"] = playerID
// 4) Determine ExternalID, PlayerName, Team cells.Each(func(i int, cell *goquery.Selection) {
// • Playoff pages use "rk", "player", "team_id" key := headers[i]
// • Regular‐season advanced uses "ranker", "name_display", "team_name_abbr" data[key] = strings.TrimSpace(cell.Text())
extID := mustAtoi(data["rk"]) if id, ok := cell.Attr("data-append-csv"); ok {
if extID == 0 { playerID = id
extID = mustAtoi(data["ranker"]) }
} })
if playerID == "" {
return // not a real data row
}
data["player-additional"] = playerID
playerName := data["player"] // 4) Determine ExternalID, PlayerName, Team.
if playerName == "" { extID := mustAtoi(data["rk"])
playerName = data["name_display"] if extID == 0 {
} extID = mustAtoi(data["ranker"])
}
teamID := data["team_id"] playerName := data["player"]
if teamID == "" { if playerName == "" {
teamID = data["team_name_abbr"] playerName = data["name_display"]
} }
// 5) Games column is always "g" in advanced (playoffs or season) teamID := data["team_id"]
g := mustAtoi(data["games"]) if teamID == "" {
if g == 0 { teamID = data["team_name_abbr"]
g = mustAtoi(data["g"]) }
}
// 6) Map into your GORM model // 5) Games column is always "g" in advanced tables.
stat := models.PlayerAdvancedStat{ g := mustAtoi(data["games"])
ExternalID: extID, if g == 0 {
PlayerID: playerID, g = mustAtoi(data["g"])
PlayerName: playerName, }
Position: data["pos"],
Age: mustAtoi(data["age"]),
Games: g,
MinutesPlayed: mustAtoi(data["mp"]),
PER: mustParseFloat(data["per"]),
TSPercent: mustParseFloat(data["ts_pct"]),
ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]),
FTR: mustParseFloat(data["fta_per_fga_pct"]),
OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
TotalRBPercent: mustParseFloat(data["trb_pct"]),
AssistPercent: mustParseFloat(data["ast_pct"]),
StealPercent: mustParseFloat(data["stl_pct"]),
BlockPercent: mustParseFloat(data["blk_pct"]),
TurnoverPercent: mustParseFloat(data["tov_pct"]),
UsagePercent: mustParseFloat(data["usg_pct"]),
OffensiveWS: mustParseFloat(data["ows"]),
DefensiveWS: mustParseFloat(data["dws"]),
WinShares: mustParseFloat(data["ws"]),
WinSharesPer: mustParseFloat(data["ws_per_48"]),
OffensiveBox: mustParseFloat(data["obpm"]),
DefensiveBox: mustParseFloat(data["dbpm"]),
Box: mustParseFloat(data["bpm"]),
VORP: mustParseFloat(data["vorp"]),
Team: teamID,
Season: season,
IsPlayoff: isPlayoff,
}
// 7) Upsert on (player_id, season, team, is_playoff) // 6) Map into your GORM model.
if err := db.Clauses(clause.OnConflict{ stat := models.PlayerAdvancedStat{
Columns: []clause.Column{ ExternalID: extID,
{Name: "player_id"}, PlayerID: playerID,
{Name: "season"}, PlayerName: playerName,
{Name: "team"}, Position: data["pos"],
{Name: "is_playoff"}, Age: mustAtoi(data["age"]),
}, Games: g,
DoUpdates: clause.AssignmentColumns([]string{ MinutesPlayed: mustAtoi(data["mp"]),
"external_id", "player_name", "position", "age", "games", PER: mustParseFloat(data["per"]),
"minutes_played", TSPercent: mustParseFloat(data["ts_pct"]),
"per", "ts_percent", "three_par", "ftr", // ← use three_par & ftr ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]),
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent", FTR: mustParseFloat(data["fta_per_fga_pct"]),
"assist_percent", "steal_percent", "block_percent", "turnover_percent", OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
"usage_percent", "offensive_ws", "defensive_ws", "win_shares", DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp", TotalRBPercent: mustParseFloat(data["trb_pct"]),
}), AssistPercent: mustParseFloat(data["ast_pct"]),
}).Create(&stat).Error; err != nil { StealPercent: mustParseFloat(data["stl_pct"]),
log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err) BlockPercent: mustParseFloat(data["blk_pct"]),
} TurnoverPercent: mustParseFloat(data["tov_pct"]),
}) UsagePercent: mustParseFloat(data["usg_pct"]),
OffensiveWS: mustParseFloat(data["ows"]),
DefensiveWS: mustParseFloat(data["dws"]),
WinShares: mustParseFloat(data["ws"]),
WinSharesPer: mustParseFloat(data["ws_per_48"]),
OffensiveBox: mustParseFloat(data["obpm"]),
DefensiveBox: mustParseFloat(data["dbpm"]),
Box: mustParseFloat(data["bpm"]),
VORP: mustParseFloat(data["vorp"]),
Team: teamID,
Season: season,
IsPlayoff: isPlayoff,
}
return nil // Add the parsed stat object to our slice.
statsToUpsert = append(statsToUpsert, stat)
})
// 7) Perform the batch upsert operation after collecting all rows.
if len(statsToUpsert) > 0 {
log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
{Name: "season"},
{Name: "team"},
{Name: "is_playoff"},
},
DoUpdates: clause.AssignmentColumns([]string{
"external_id", "player_name", "position", "age", "games",
"minutes_played", "per", "ts_percent", "three_par", "ftr",
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
}),
}).Create(&statsToUpsert).Error; err != nil {
log.Printf("Failed to batch upsert advanced player stats: %v", err)
return err
}
log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season)
} else {
log.Printf("No advanced player data found to import for season %d.", season)
}
// --- BATCHING LOGIC END ---
return nil
} }