nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 05:55:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

batch insert adv

Browse Source
This commit is contained in:
Ravi Prasad
2025-06-19 22:20:43 -05:00
parent 1a2a4c1b73
commit e6851b3bb6
3 changed files with 211 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
import.go
+24 -24
View File
@@ -9,33 +9,33 @@ import (
"github.com/nprasad2077/NBA_Go/utils"
)
// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
// func importPlayerAdvanced(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err)
// }
// log.Printf("Advanced import for season: %d", season)
// time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1000 * time.Millisecond)
// }
// }
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2000; season <= 2014; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
}
// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err)
// }
// log.Printf("Advanced Playoffs import for season: %d", season)
// time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1250 * time.Millisecond)
// }
// }
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2000; season <= 2014; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced Playoffs import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
utils.SleepWithJitter(1250 * time.Millisecond)
}
}
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) {
for season := 2013; season <= 2014; season++ {
for season := 2000; season <= 2012; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2013; season <= 2014; season++ {
for season := 2000; season <= 2012; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
main.go
+4 -4
View File
@@ -45,11 +45,11 @@ func main() {
// Run all migrations + import steps exactly once
db := config.InitDB(true)
// importPlayerAdvanced(db)
// log.Println("🎉 Player Advanced Import completed successfully")
importPlayerAdvanced(db)
log.Println("🎉 Player Advanced Import completed successfully")
// importPlayerAdvancedPlayoffs(db)
// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
importPlayerAdvancedPlayoffs(db)
log.Println("🎉 Player Advanced Playoffs Import completed successfully")
importPlayerTotalsScrape(db)
log.Println("🎉 Player Totals (scraped) Import completed successfully")
services/player_advanced_playoffs_scrape_service.go
+34 -27
View File
@@ -29,8 +29,8 @@ func urlForAdvSeason(season int, isPlayoff bool) string {
return fmt.Sprintf(advancedURLFmt, season)
}
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table
// (regular or playoffs) and upserts into the PlayerAdvancedStat model.
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs)
// and batch upserts the data into the PlayerAdvancedStat model.
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
url := urlForAdvSeason(season, isPlayoff)
req, err := http.NewRequest("GET", url, nil)
@@ -54,9 +54,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
return err
}
// 1) Determine parent wrapper and table selector
// - Regular season: <div id="all_advanced"><!-- <table id="advanced">…</table> --></div>
// - Playoffs: <div id="all_advanced_stats"><!-- <table id="advanced_stats">…</table> --></div>
// 1) Determine parent wrapper and table selector. This logic is complex because
// the table is often hidden inside an HTML comment.
var parentDivSelector, tableSelector string
if isPlayoff {
parentDivSelector = "#all_advanced_stats"
@@ -66,10 +65,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
tableSelector = "table#advanced"
}
// Attempt to find the table node directly (it will be inside a comment for regular season)
table := doc.Find(parentDivSelector + " " + tableSelector)
if table.Length() == 0 {
// Look for the commented‐out HTML inside parentDivSelector
commentSel := doc.
Find(parentDivSelector).
Contents().
@@ -78,32 +75,34 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
})
if commentSel.Length() == 0 {
return fmt.Errorf("could not find advanced stats table (even inside comment)")
return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season)
}
// Extract the raw HTML string from the comment, then re‐parse
commentedHTML := commentSel.Nodes[0].FirstChild.Data
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
if err != nil {
return fmt.Errorf("failed to parse commented advanced HTML: %w", err)
return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err)
}
table = innerDoc.Find(tableSelector)
if table.Length() == 0 {
return fmt.Errorf("could not find advanced stats table after un‐commenting")
return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season)
}
}
// 2) Collect the data-stat keys in header order
// 2) Collect the data-stat keys in header order.
var headers []string
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
headers = append(headers, stat)
}
})
// Add our appended‐player column
headers = append(headers, "player-additional")
// 3) Iterate each row
// --- BATCHING LOGIC START ---
// Create a slice to hold all the player stats parsed from the page.
var statsToUpsert []models.PlayerAdvancedStat
// 3) Iterate each row and collect player stats into the slice.
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
if tr.HasClass("thead") {
return // skip repeated header rows
@@ -120,14 +119,11 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
}
})
if playerID == "" {
// not a real data row
return
return // not a real data row
}
data["player-additional"] = playerID
// 4) Determine ExternalID, PlayerName, Team
// • Playoff pages use "rk", "player", "team_id"
// • Regular‐season advanced uses "ranker", "name_display", "team_name_abbr"
// 4) Determine ExternalID, PlayerName, Team.
extID := mustAtoi(data["rk"])
if extID == 0 {
extID = mustAtoi(data["ranker"])
@@ -143,13 +139,13 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
teamID = data["team_name_abbr"]
}
// 5) Games column is always "g" in advanced (playoffs or season)
// 5) Games column is always "g" in advanced tables.
g := mustAtoi(data["games"])
if g == 0 {
g = mustAtoi(data["g"])
}
// 6) Map into your GORM model
// 6) Map into your GORM model.
stat := models.PlayerAdvancedStat{
ExternalID: extID,
PlayerID: playerID,
@@ -183,7 +179,14 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
IsPlayoff: isPlayoff,
}
// 7) Upsert on (player_id, season, team, is_playoff)
// Add the parsed stat object to our slice.
statsToUpsert = append(statsToUpsert, stat)
})
// 7) Perform the batch upsert operation after collecting all rows.
if len(statsToUpsert) > 0 {
log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
@@ -193,17 +196,21 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
},
DoUpdates: clause.AssignmentColumns([]string{
"external_id", "player_name", "position", "age", "games",
"minutes_played",
"per", "ts_percent", "three_par", "ftr", // ← use three_par & ftr
"minutes_played", "per", "ts_percent", "three_par", "ftr",
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
}),
}).Create(&stat).Error; err != nil {
log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
}).Create(&statsToUpsert).Error; err != nil {
log.Printf("Failed to batch upsert advanced player stats: %v", err)
return err
}
})
log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season)
} else {
log.Printf("No advanced player data found to import for season %d.", season)
}
// --- BATCHING LOGIC END ---
return nil
}