mirror of
https://github.com/nprasad2077/NBA_Go.git
synced 2026-09-22 05:55:13 +00:00
batch insert totals
This commit is contained in:
Ravi Prasad
@@ -9,33 +9,33 @@ import (
|
||||
"github.com/nprasad2077/NBA_Go/utils"
|
||||
)
|
||||
|
||||
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
|
||||
func importPlayerAdvanced(db *gorm.DB) {
|
||||
for season := 2015; season <= 2019; season++ {
|
||||
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
|
||||
log.Printf("advanced import failed for %d: %v", season, err)
|
||||
}
|
||||
log.Printf("Advanced import for season: %d", season)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
utils.SleepWithJitter(1000 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
// // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
|
||||
// func importPlayerAdvanced(db *gorm.DB) {
|
||||
// for season := 2013; season <= 2014; season++ {
|
||||
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
|
||||
// log.Printf("advanced import failed for %d: %v", season, err)
|
||||
// }
|
||||
// log.Printf("Advanced import for season: %d", season)
|
||||
// time.Sleep(1100 * time.Millisecond)
|
||||
// utils.SleepWithJitter(1000 * time.Millisecond)
|
||||
// }
|
||||
// }
|
||||
|
||||
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
|
||||
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
|
||||
for season := 2015; season <= 2019; season++ {
|
||||
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
|
||||
log.Printf("advanced import failed for %d: %v", season, err)
|
||||
}
|
||||
log.Printf("Advanced Playoffs import for season: %d", season)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
utils.SleepWithJitter(1250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
// // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
|
||||
// func importPlayerAdvancedPlayoffs(db *gorm.DB) {
|
||||
// for season := 2013; season <= 2014; season++ {
|
||||
// if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
|
||||
// log.Printf("advanced import failed for %d: %v", season, err)
|
||||
// }
|
||||
// log.Printf("Advanced Playoffs import for season: %d", season)
|
||||
// time.Sleep(1100 * time.Millisecond)
|
||||
// utils.SleepWithJitter(1250 * time.Millisecond)
|
||||
// }
|
||||
// }
|
||||
|
||||
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
|
||||
func importPlayerTotalsScrape(db *gorm.DB) {
|
||||
for season := 2015; season <= 2019; season++ {
|
||||
for season := 2013; season <= 2014; season++ {
|
||||
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
|
||||
log.Printf("scraped totals import failed for %d: %v", season, err)
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
|
||||
|
||||
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
|
||||
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
|
||||
for season := 2015; season <= 2019; season++ {
|
||||
for season := 2013; season <= 2014; season++ {
|
||||
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
|
||||
log.Printf("scraped playoffs import failed for %d: %v", season, err)
|
||||
}
|
||||
|
||||
@@ -45,11 +45,11 @@ func main() {
|
||||
// Run all migrations + import steps exactly once
|
||||
db := config.InitDB(true)
|
||||
|
||||
importPlayerAdvanced(db)
|
||||
log.Println("🎉 Player Advanced Import completed successfully")
|
||||
// importPlayerAdvanced(db)
|
||||
// log.Println("🎉 Player Advanced Import completed successfully")
|
||||
|
||||
importPlayerAdvancedPlayoffs(db)
|
||||
log.Println("🎉 Player Advanced Playoffs Import completed successfully")
|
||||
// importPlayerAdvancedPlayoffs(db)
|
||||
// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
|
||||
|
||||
importPlayerTotalsScrape(db)
|
||||
log.Println("🎉 Player Totals (scraped) Import completed successfully")
|
||||
|
||||
@@ -3,209 +3,205 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/nprasad2077/NBA_Go/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/nprasad2077/NBA_Go/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html"
|
||||
playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html"
|
||||
regularURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_totals.html"
|
||||
playoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_totals.html"
|
||||
)
|
||||
|
||||
// urlForSeason chooses regular vs. playoff URL.
|
||||
func urlForSeason(season int, isPlayoff bool) string {
|
||||
if isPlayoff {
|
||||
return fmt.Sprintf(playoffURLFmt, season)
|
||||
}
|
||||
return fmt.Sprintf(regularURLFmt, season)
|
||||
if isPlayoff {
|
||||
return fmt.Sprintf(playoffURLFmt, season)
|
||||
}
|
||||
return fmt.Sprintf(regularURLFmt, season)
|
||||
}
|
||||
|
||||
// FetchAndStorePlayerTotalScrapedStats scrapes BR totals (regular or playoffs)
|
||||
// and upserts into PlayerTotalStat.
|
||||
// and batch upserts them into PlayerTotalStat for significantly better performance.
|
||||
func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
|
||||
url := urlForSeason(season, isPlayoff)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
url := urlForSeason(season, isPlayoff)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table := doc.Find("table#totals_stats")
|
||||
if table.Length() == 0 {
|
||||
return fmt.Errorf("could not find table#totals_stats")
|
||||
}
|
||||
table := doc.Find("table#totals_stats")
|
||||
if table.Length() == 0 {
|
||||
return fmt.Errorf("could not find table#totals_stats for season %d", season)
|
||||
}
|
||||
|
||||
// table := doc.Find("table#totals_stats")
|
||||
// if table.Length() == 0 {
|
||||
// commentSel := doc.
|
||||
// Find("div#all_totals_stats").
|
||||
// Contents().
|
||||
// FilterFunction(func(i int, s *goquery.Selection) bool {
|
||||
// return goquery.NodeName(s) == "#comment"
|
||||
// })
|
||||
// 1) Collect the data-stat keys in header order.
|
||||
var headers []string
|
||||
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
|
||||
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
|
||||
headers = append(headers, stat)
|
||||
}
|
||||
})
|
||||
headers = append(headers, "player-additional") // Add the appended-player column.
|
||||
|
||||
// if commentSel.Length() == 0 {
|
||||
// return fmt.Errorf("could not find table#totals_stats (even inside comment)")
|
||||
// }
|
||||
// --- BATCHING LOGIC START ---
|
||||
// Create a slice to hold all the player stats parsed from the page.
|
||||
var statsToUpsert []models.PlayerTotalStat
|
||||
|
||||
// commentedHTML := commentSel.Nodes[0].FirstChild.Data
|
||||
// innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
|
||||
// if err != nil {
|
||||
// return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err)
|
||||
// }
|
||||
// table = innerDoc.Find("table#totals_stats")
|
||||
// if table.Length() == 0 {
|
||||
// return fmt.Errorf("could not find table#totals_stats after un‐commenting")
|
||||
// }
|
||||
// }
|
||||
// 2) Iterate rows and collect all player stats into the slice.
|
||||
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
|
||||
return // Skip repeated header rows inside the table body.
|
||||
}
|
||||
|
||||
// 1) collect the data-stat keys in header order
|
||||
var headers []string
|
||||
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
|
||||
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
|
||||
headers = append(headers, stat)
|
||||
}
|
||||
})
|
||||
// add the appended-player column
|
||||
headers = append(headers, "player-additional")
|
||||
cells := tr.Find("th, td")
|
||||
data := make(map[string]string, len(headers))
|
||||
var playerID string
|
||||
|
||||
// 2) iterate rows
|
||||
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
if cl, _ := tr.Attr("class"); strings.Contains(cl, "thead") {
|
||||
return // skip header rows
|
||||
}
|
||||
cells.Each(func(i int, cell *goquery.Selection) {
|
||||
text := strings.TrimSpace(cell.Text())
|
||||
key := headers[i]
|
||||
data[key] = text
|
||||
if id, ok := cell.Attr("data-append-csv"); ok {
|
||||
playerID = id
|
||||
}
|
||||
})
|
||||
|
||||
cells := tr.Find("th, td")
|
||||
data := make(map[string]string, len(headers))
|
||||
var playerID string
|
||||
// Skip rows that aren't actual player data rows.
|
||||
if playerID == "" {
|
||||
return
|
||||
}
|
||||
data["player-additional"] = playerID
|
||||
|
||||
cells.Each(func(i int, cell *goquery.Selection) {
|
||||
text := strings.TrimSpace(cell.Text())
|
||||
key := headers[i]
|
||||
data[key] = text
|
||||
if id, ok := cell.Attr("data-append-csv"); ok {
|
||||
playerID = id
|
||||
}
|
||||
})
|
||||
if playerID == "" {
|
||||
return
|
||||
}
|
||||
data["player-additional"] = playerID
|
||||
// 3a) Pick the right “ExternalID” key.
|
||||
extID := mustAtoi(data["rk"])
|
||||
if extID == 0 {
|
||||
extID = mustAtoi(data["ranker"])
|
||||
}
|
||||
|
||||
// 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”)
|
||||
extID := mustAtoi(data["rk"])
|
||||
if extID == 0 {
|
||||
extID = mustAtoi(data["ranker"])
|
||||
}
|
||||
// 3b) Pick the right “PlayerName” key.
|
||||
playerName := data["player"]
|
||||
if playerName == "" {
|
||||
playerName = data["name_display"]
|
||||
}
|
||||
|
||||
// 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”)
|
||||
playerName := data["player"]
|
||||
if playerName == "" {
|
||||
playerName = data["name_display"]
|
||||
}
|
||||
// 3c) Pick the right “Team” key.
|
||||
teamID := data["team_id"]
|
||||
if teamID == "" {
|
||||
teamID = data["team_name_abbr"]
|
||||
}
|
||||
|
||||
// 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”)
|
||||
teamID := data["team_id"]
|
||||
if teamID == "" {
|
||||
teamID = data["team_name_abbr"]
|
||||
}
|
||||
// Pick “games”.
|
||||
g := mustAtoi(data["games"])
|
||||
if g == 0 {
|
||||
g = mustAtoi(data["g"])
|
||||
}
|
||||
|
||||
// pick “games” → fallback to “g” if empty
|
||||
g := mustAtoi(data["games"])
|
||||
if g == 0 {
|
||||
g = mustAtoi(data["g"])
|
||||
}
|
||||
// Pick “games_started”.
|
||||
gs := mustAtoi(data["games_started"])
|
||||
if gs == 0 {
|
||||
gs = mustAtoi(data["gs"])
|
||||
}
|
||||
|
||||
// pick “games_started” → fallback to “gs” if empty
|
||||
gs := mustAtoi(data["games_started"])
|
||||
if gs == 0 {
|
||||
gs = mustAtoi(data["gs"])
|
||||
}
|
||||
stat := models.PlayerTotalStat{
|
||||
ExternalID: extID,
|
||||
PlayerID: playerID,
|
||||
PlayerName: playerName,
|
||||
Position: data["pos"],
|
||||
Age: mustAtoi(data["age"]),
|
||||
Games: g,
|
||||
GamesStarted: gs,
|
||||
MinutesPG: mustParseFloat(data["mp"]),
|
||||
FieldGoals: mustAtoi(data["fg"]),
|
||||
FieldAttempts: mustAtoi(data["fga"]),
|
||||
FieldPercent: mustParseFloat(data["fg_pct"]),
|
||||
ThreeFG: mustAtoi(data["fg3"]),
|
||||
ThreeAttempts: mustAtoi(data["fg3a"]),
|
||||
ThreePercent: mustParseFloat(data["fg3_pct"]),
|
||||
TwoFG: mustAtoi(data["fg2"]),
|
||||
TwoAttempts: mustAtoi(data["fg2a"]),
|
||||
TwoPercent: mustParseFloat(data["fg2_pct"]),
|
||||
EffectFGPercent: mustParseFloat(data["efg_pct"]),
|
||||
FT: mustAtoi(data["ft"]),
|
||||
FTAttempts: mustAtoi(data["fta"]),
|
||||
FTPercent: mustParseFloat(data["ft_pct"]),
|
||||
OffensiveRB: mustAtoi(data["orb"]),
|
||||
DefensiveRB: mustAtoi(data["drb"]),
|
||||
TotalRB: mustAtoi(data["trb"]),
|
||||
Assists: mustAtoi(data["ast"]),
|
||||
Steals: mustAtoi(data["stl"]),
|
||||
Blocks: mustAtoi(data["blk"]),
|
||||
Turnovers: mustAtoi(data["tov"]),
|
||||
PersonalFouls: mustAtoi(data["pf"]),
|
||||
Points: mustAtoi(data["pts"]),
|
||||
Team: teamID,
|
||||
Season: season,
|
||||
IsPlayoff: isPlayoff,
|
||||
}
|
||||
|
||||
stat := models.PlayerTotalStat{
|
||||
ExternalID: extID,
|
||||
PlayerID: playerID,
|
||||
PlayerName: playerName,
|
||||
Position: data["pos"],
|
||||
Age: mustAtoi(data["age"]),
|
||||
Games: g,
|
||||
GamesStarted: gs,
|
||||
MinutesPG: mustParseFloat(data["mp"]),
|
||||
FieldGoals: mustAtoi(data["fg"]),
|
||||
FieldAttempts: mustAtoi(data["fga"]),
|
||||
FieldPercent: mustParseFloat(data["fg_pct"]),
|
||||
ThreeFG: mustAtoi(data["fg3"]),
|
||||
ThreeAttempts: mustAtoi(data["fg3a"]),
|
||||
ThreePercent: mustParseFloat(data["fg3_pct"]),
|
||||
TwoFG: mustAtoi(data["fg2"]),
|
||||
TwoAttempts: mustAtoi(data["fg2a"]),
|
||||
TwoPercent: mustParseFloat(data["fg2_pct"]),
|
||||
EffectFGPercent: mustParseFloat(data["efg_pct"]),
|
||||
FT: mustAtoi(data["ft"]),
|
||||
FTAttempts: mustAtoi(data["fta"]),
|
||||
FTPercent: mustParseFloat(data["ft_pct"]),
|
||||
OffensiveRB: mustAtoi(data["orb"]),
|
||||
DefensiveRB: mustAtoi(data["drb"]),
|
||||
TotalRB: mustAtoi(data["trb"]),
|
||||
Assists: mustAtoi(data["ast"]),
|
||||
Steals: mustAtoi(data["stl"]),
|
||||
Blocks: mustAtoi(data["blk"]),
|
||||
Turnovers: mustAtoi(data["tov"]),
|
||||
PersonalFouls: mustAtoi(data["pf"]),
|
||||
Points: mustAtoi(data["pts"]),
|
||||
Team: teamID,
|
||||
Season: season,
|
||||
IsPlayoff: isPlayoff,
|
||||
}
|
||||
// Add the parsed stat object to our slice instead of writing to the DB immediately.
|
||||
statsToUpsert = append(statsToUpsert, stat)
|
||||
})
|
||||
|
||||
// 4) upsert on (player_id, season, team, is_playoff)
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "player_id"},
|
||||
{Name: "season"},
|
||||
{Name: "team"},
|
||||
{Name: "is_playoff"},
|
||||
},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"external_id", "player_name", "position", "age",
|
||||
"games", "games_started", "minutes_pg",
|
||||
"field_goals", "field_attempts", "field_percent",
|
||||
"three_fg", "three_attempts", "three_percent",
|
||||
"two_fg", "two_attempts", "two_percent",
|
||||
"effect_fg_percent",
|
||||
"ft", "ft_attempts", "ft_percent",
|
||||
"offensive_rb", "defensive_rb", "total_rb",
|
||||
"assists", "steals", "blocks", "turnovers",
|
||||
"personal_fouls", "points",
|
||||
}),
|
||||
}).Create(&stat).Error; err != nil {
|
||||
log.Printf("Failed upsert for %s: %v", stat.PlayerID, err)
|
||||
}
|
||||
})
|
||||
// 3) Perform the batch upsert operation after collecting all rows.
|
||||
if len(statsToUpsert) > 0 {
|
||||
log.Printf("Attempting to batch upsert %d player total stats for season %d...", len(statsToUpsert), season)
|
||||
|
||||
return nil
|
||||
// GORM's OnConflict clause works with slices, performing the batch operation efficiently.
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "player_id"},
|
||||
{Name: "season"},
|
||||
{Name: "team"},
|
||||
{Name: "is_playoff"},
|
||||
},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"external_id", "player_name", "position", "age",
|
||||
"games", "games_started", "minutes_pg",
|
||||
"field_goals", "field_attempts", "field_percent",
|
||||
"three_fg", "three_attempts", "three_percent",
|
||||
"two_fg", "two_attempts", "two_percent",
|
||||
"effect_fg_percent",
|
||||
"ft", "ft_attempts", "ft_percent",
|
||||
"offensive_rb", "defensive_rb", "total_rb",
|
||||
"assists", "steals", "blocks", "turnovers",
|
||||
"personal_fouls", "points",
|
||||
}),
|
||||
}).Create(&statsToUpsert).Error; err != nil {
|
||||
// If the batch operation fails, log the error and return it.
|
||||
log.Printf("Failed to batch upsert player total stats: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("✅ Successfully batch upserted %d records for season %d.", len(statsToUpsert), season)
|
||||
} else {
|
||||
log.Printf("No player data found to import for season %d.", season)
|
||||
}
|
||||
// --- BATCHING LOGIC END ---
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user