nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

batch insert totals

Browse Source
This commit is contained in:
Ravi Prasad
2025-06-19 22:13:11 -05:00
parent d8e3ff34b6
commit 1a2a4c1b73
3 changed files with 199 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
import.go
+24 -24
View File
@@ -9,33 +9,33 @@ import (
"github.com/nprasad2077/NBA_Go/utils"
)
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2015; season <= 2019; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
}
// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
// func importPlayerAdvanced(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err)
// }
// log.Printf("Advanced import for season: %d", season)
// time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1000 * time.Millisecond)
// }
// }
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2015; season <= 2019; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced Playoffs import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
utils.SleepWithJitter(1250 * time.Millisecond)
}
}
// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
// for season := 2013; season <= 2014; season++ {
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
// log.Printf("advanced import failed for %d: %v", season, err)
// }
// log.Printf("Advanced Playoffs import for season: %d", season)
// time.Sleep(1100 * time.Millisecond)
// utils.SleepWithJitter(1250 * time.Millisecond)
// }
// }
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) {
for season := 2015; season <= 2019; season++ {
for season := 2013; season <= 2014; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2015; season <= 2019; season++ {
for season := 2013; season <= 2014; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
main.go
+4 -4
View File
@@ -45,11 +45,11 @@ func main() {
// Run all migrations + import steps exactly once
db := config.InitDB(true)
importPlayerAdvanced(db)
log.Println("🎉 Player Advanced Import completed successfully")
// importPlayerAdvanced(db)
// log.Println("🎉 Player Advanced Import completed successfully")
importPlayerAdvancedPlayoffs(db)
log.Println("🎉 Player Advanced Playoffs Import completed successfully")
// importPlayerAdvancedPlayoffs(db)
// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
importPlayerTotalsScrape(db)
log.Println("🎉 Player Totals (scraped) Import completed successfully")
services/player_total_scrape_service.go
+36 -40
View File
@@ -30,7 +30,7 @@ func urlForSeason(season int, isPlayoff bool) string {
}
// FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs)
// and upserts into PlayerTotalStat.
// and batch upserts them into PlayerTotalStat for significantly better performance.
func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
url := urlForSeason(season, isPlayoff)
req, err := http.NewRequest("GET", url, nil)
@@ -56,47 +56,26 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
table := doc.Find("table#totals_stats")
if table.Length() == 0 {
return fmt.Errorf("could not find table#totals_stats")
return fmt.Errorf("could not find table#totals_stats for season %d", season)
}
// table := doc.Find("table#totals_stats")
// if table.Length() == 0 {
// commentSel := doc.
// Find("div#all_totals_stats").
// Contents().
// FilterFunction(func(i int, s *goquery.Selection) bool {
// return goquery.NodeName(s) == "#comment"
// })
// if commentSel.Length() == 0 {
// return fmt.Errorf("could not find table#totals_stats (even inside comment)")
// }
// commentedHTML := commentSel.Nodes[0].FirstChild.Data
// innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
// if err != nil {
// return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err)
// }
// table = innerDoc.Find("table#totals_stats")
// if table.Length() == 0 {
// return fmt.Errorf("could not find table#totals_stats after un‐commenting")
// }
// }
// 1) collect the data-stat keys in header order
// 1) Collect the data-stat keys in header order.
var headers []string
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
headers = append(headers, stat)
}
})
// add the appended-player column
headers = append(headers, "player-additional")
headers = append(headers, "player-additional") // Add the appended-player column.
// 2) iterate rows
// --- BATCHING LOGIC START ---
// Create a slice to hold all the player stats parsed from the page.
var statsToUpsert []models.PlayerTotalStat
// 2) Iterate rows and collect all player stats into the slice.
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
return // skip header rows
return // Skip repeated header rows inside the table body.
}
cells := tr.Find("th, td")
@@ -111,36 +90,38 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
playerID = id
}
})
// Skip rows that aren't actual player data rows.
if playerID == "" {
return
}
data["player-additional"] = playerID
// 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”)
// 3a) Pick the right “ExternalID” key.
extID := mustAtoi(data["rk"])
if extID == 0 {
extID = mustAtoi(data["ranker"])
}
// 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”)
// 3b) Pick the right “PlayerName” key.
playerName := data["player"]
if playerName == "" {
playerName = data["name_display"]
}
// 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”)
// 3c) Pick the right “Team” key.
teamID := data["team_id"]
if teamID == "" {
teamID = data["team_name_abbr"]
}
// pick “games” → fallback to “g” if empty
// Pick “games”.
g := mustAtoi(data["games"])
if g == 0 {
g = mustAtoi(data["g"])
}
// pick “games_started” → fallback to “gs” if empty
// Pick “games_started”.
gs := mustAtoi(data["games_started"])
if gs == 0 {
gs = mustAtoi(data["gs"])
@@ -182,7 +163,15 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
IsPlayoff: isPlayoff,
}
// 4) upsert on (player_id, season, team, is_playoff)
// Add the parsed stat object to our slice instead of writing to the DB immediately.
statsToUpsert = append(statsToUpsert, stat)
})
// 3) Perform the batch upsert operation after collecting all rows.
if len(statsToUpsert) > 0 {
log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season)
// GORM's OnConflict clause works with slices, performing the batch operation efficiently.
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
@@ -202,10 +191,17 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
"assists", "steals", "blocks", "turnovers",
"personal_fouls", "points",
}),
}).Create(&stat).Error; err != nil {
log.Printf("Failed upsert for %s: %v", stat.PlayerID, err)
}).Create(&statsToUpsert).Error; err != nil {
// If the batch operation fails, log the error and return it.
log.Printf("Failed to batch upsert player total stats: %v", err)
return err
}
})
log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season)
} else {
log.Printf("No player data found to import for season %d.", season)
}
// --- BATCHING LOGIC END ---
return nil
}