mirror of
https://github.com/nprasad2077/NBA_Go.git
synced 2026-09-22 14:05:13 +00:00
batch insert adv
This commit is contained in:
Ravi Prasad
@@ -3,207 +3,214 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/nprasad2077/NBA_Go/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/nprasad2077/NBA_Go/models"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
|
||||
advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
|
||||
advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
|
||||
advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
|
||||
)
|
||||
|
||||
// urlForAdvSeason picks the correct URL based on isPlayoff.
|
||||
func urlForAdvSeason(season int, isPlayoff bool) string {
|
||||
if isPlayoff {
|
||||
return fmt.Sprintf(advancedPlayoffURLFmt, season)
|
||||
}
|
||||
return fmt.Sprintf(advancedURLFmt, season)
|
||||
if isPlayoff {
|
||||
return fmt.Sprintf(advancedPlayoffURLFmt, season)
|
||||
}
|
||||
return fmt.Sprintf(advancedURLFmt, season)
|
||||
}
|
||||
|
||||
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table
|
||||
// (regular or playoffs) and upserts into the PlayerAdvancedStat model.
|
||||
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table (regular or playoffs)
|
||||
// and batch upserts the data into the PlayerAdvancedStat model.
|
||||
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
|
||||
url := urlForAdvSeason(season, isPlayoff)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
url := urlForAdvSeason(season, isPlayoff)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
htmlBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
htmlBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 1) Determine parent wrapper and table selector
|
||||
// - Regular season: <div id="all_advanced"><!-- <table id="advanced">…</table> --></div>
|
||||
// - Playoffs: <div id="all_advanced_stats"><!-- <table id="advanced_stats">…</table> --></div>
|
||||
var parentDivSelector, tableSelector string
|
||||
if isPlayoff {
|
||||
parentDivSelector = "#all_advanced_stats"
|
||||
tableSelector = "table#advanced_stats"
|
||||
} else {
|
||||
parentDivSelector = "#all_advanced"
|
||||
tableSelector = "table#advanced"
|
||||
}
|
||||
// 1) Determine parent wrapper and table selector. This logic is complex because
|
||||
// the table is often hidden inside an HTML comment.
|
||||
var parentDivSelector, tableSelector string
|
||||
if isPlayoff {
|
||||
parentDivSelector = "#all_advanced_stats"
|
||||
tableSelector = "table#advanced_stats"
|
||||
} else {
|
||||
parentDivSelector = "#all_advanced"
|
||||
tableSelector = "table#advanced"
|
||||
}
|
||||
|
||||
// Attempt to find the table node directly (it will be inside a comment for regular season)
|
||||
table := doc.Find(parentDivSelector + " " + tableSelector)
|
||||
if table.Length() == 0 {
|
||||
// Look for the commented‐out HTML inside parentDivSelector
|
||||
commentSel := doc.
|
||||
Find(parentDivSelector).
|
||||
Contents().
|
||||
FilterFunction(func(i int, s *goquery.Selection) bool {
|
||||
return goquery.NodeName(s) == "#comment"
|
||||
})
|
||||
table := doc.Find(parentDivSelector + " " + tableSelector)
|
||||
if table.Length() == 0 {
|
||||
commentSel := doc.
|
||||
Find(parentDivSelector).
|
||||
Contents().
|
||||
FilterFunction(func(i int, s *goquery.Selection) bool {
|
||||
return goquery.NodeName(s) == "#comment"
|
||||
})
|
||||
|
||||
if commentSel.Length() == 0 {
|
||||
return fmt.Errorf("could not find advanced stats table (even inside comment)")
|
||||
}
|
||||
if commentSel.Length() == 0 {
|
||||
return fmt.Errorf("could not find advanced stats table (even inside comment) for season %d", season)
|
||||
}
|
||||
|
||||
// Extract the raw HTML string from the comment, then re‐parse
|
||||
commentedHTML := commentSel.Nodes[0].FirstChild.Data
|
||||
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse commented advanced HTML: %w", err)
|
||||
}
|
||||
table = innerDoc.Find(tableSelector)
|
||||
if table.Length() == 0 {
|
||||
return fmt.Errorf("could not find advanced stats table after un‐commenting")
|
||||
}
|
||||
}
|
||||
commentedHTML := commentSel.Nodes[0].FirstChild.Data
|
||||
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse commented advanced HTML for season %d: %w", season, err)
|
||||
}
|
||||
table = innerDoc.Find(tableSelector)
|
||||
if table.Length() == 0 {
|
||||
return fmt.Errorf("could not find advanced stats table after un-commenting for season %d", season)
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Collect the data-stat keys in header order
|
||||
var headers []string
|
||||
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
|
||||
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
|
||||
headers = append(headers, stat)
|
||||
}
|
||||
})
|
||||
// Add our appended‐player column
|
||||
headers = append(headers, "player-additional")
|
||||
// 2) Collect the data-stat keys in header order.
|
||||
var headers []string
|
||||
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
|
||||
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
|
||||
headers = append(headers, stat)
|
||||
}
|
||||
})
|
||||
headers = append(headers, "player-additional")
|
||||
|
||||
// 3) Iterate each row
|
||||
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
if tr.HasClass("thead") {
|
||||
return // skip repeated header rows
|
||||
}
|
||||
cells := tr.Find("th, td")
|
||||
data := make(map[string]string, len(headers))
|
||||
var playerID string
|
||||
// --- BATCHING LOGIC START ---
|
||||
// Create a slice to hold all the player stats parsed from the page.
|
||||
var statsToUpsert []models.PlayerAdvancedStat
|
||||
|
||||
cells.Each(func(i int, cell *goquery.Selection) {
|
||||
key := headers[i]
|
||||
data[key] = strings.TrimSpace(cell.Text())
|
||||
if id, ok := cell.Attr("data-append-csv"); ok {
|
||||
playerID = id
|
||||
}
|
||||
})
|
||||
if playerID == "" {
|
||||
// not a real data row
|
||||
return
|
||||
}
|
||||
data["player-additional"] = playerID
|
||||
// 3) Iterate each row and collect player stats into the slice.
|
||||
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
if tr.HasClass("thead") {
|
||||
return // skip repeated header rows
|
||||
}
|
||||
cells := tr.Find("th, td")
|
||||
data := make(map[string]string, len(headers))
|
||||
var playerID string
|
||||
|
||||
// 4) Determine ExternalID, PlayerName, Team
|
||||
// • Playoff pages use "rk", "player", "team_id"
|
||||
// • Regular‐season advanced uses "ranker", "name_display", "team_name_abbr"
|
||||
extID := mustAtoi(data["rk"])
|
||||
if extID == 0 {
|
||||
extID = mustAtoi(data["ranker"])
|
||||
}
|
||||
cells.Each(func(i int, cell *goquery.Selection) {
|
||||
key := headers[i]
|
||||
data[key] = strings.TrimSpace(cell.Text())
|
||||
if id, ok := cell.Attr("data-append-csv"); ok {
|
||||
playerID = id
|
||||
}
|
||||
})
|
||||
if playerID == "" {
|
||||
return // not a real data row
|
||||
}
|
||||
data["player-additional"] = playerID
|
||||
|
||||
playerName := data["player"]
|
||||
if playerName == "" {
|
||||
playerName = data["name_display"]
|
||||
}
|
||||
// 4) Determine ExternalID, PlayerName, Team.
|
||||
extID := mustAtoi(data["rk"])
|
||||
if extID == 0 {
|
||||
extID = mustAtoi(data["ranker"])
|
||||
}
|
||||
|
||||
teamID := data["team_id"]
|
||||
if teamID == "" {
|
||||
teamID = data["team_name_abbr"]
|
||||
}
|
||||
playerName := data["player"]
|
||||
if playerName == "" {
|
||||
playerName = data["name_display"]
|
||||
}
|
||||
|
||||
// 5) Games column is always "g" in advanced (playoffs or season)
|
||||
g := mustAtoi(data["games"])
|
||||
if g == 0 {
|
||||
g = mustAtoi(data["g"])
|
||||
}
|
||||
teamID := data["team_id"]
|
||||
if teamID == "" {
|
||||
teamID = data["team_name_abbr"]
|
||||
}
|
||||
|
||||
// 6) Map into your GORM model
|
||||
stat := models.PlayerAdvancedStat{
|
||||
ExternalID: extID,
|
||||
PlayerID: playerID,
|
||||
PlayerName: playerName,
|
||||
Position: data["pos"],
|
||||
Age: mustAtoi(data["age"]),
|
||||
Games: g,
|
||||
MinutesPlayed: mustAtoi(data["mp"]),
|
||||
PER: mustParseFloat(data["per"]),
|
||||
TSPercent: mustParseFloat(data["ts_pct"]),
|
||||
ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]),
|
||||
FTR: mustParseFloat(data["fta_per_fga_pct"]),
|
||||
OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
|
||||
DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
|
||||
TotalRBPercent: mustParseFloat(data["trb_pct"]),
|
||||
AssistPercent: mustParseFloat(data["ast_pct"]),
|
||||
StealPercent: mustParseFloat(data["stl_pct"]),
|
||||
BlockPercent: mustParseFloat(data["blk_pct"]),
|
||||
TurnoverPercent: mustParseFloat(data["tov_pct"]),
|
||||
UsagePercent: mustParseFloat(data["usg_pct"]),
|
||||
OffensiveWS: mustParseFloat(data["ows"]),
|
||||
DefensiveWS: mustParseFloat(data["dws"]),
|
||||
WinShares: mustParseFloat(data["ws"]),
|
||||
WinSharesPer: mustParseFloat(data["ws_per_48"]),
|
||||
OffensiveBox: mustParseFloat(data["obpm"]),
|
||||
DefensiveBox: mustParseFloat(data["dbpm"]),
|
||||
Box: mustParseFloat(data["bpm"]),
|
||||
VORP: mustParseFloat(data["vorp"]),
|
||||
Team: teamID,
|
||||
Season: season,
|
||||
IsPlayoff: isPlayoff,
|
||||
}
|
||||
// 5) Games column is always "g" in advanced tables.
|
||||
g := mustAtoi(data["games"])
|
||||
if g == 0 {
|
||||
g = mustAtoi(data["g"])
|
||||
}
|
||||
|
||||
// 7) Upsert on (player_id, season, team, is_playoff)
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "player_id"},
|
||||
{Name: "season"},
|
||||
{Name: "team"},
|
||||
{Name: "is_playoff"},
|
||||
},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"external_id", "player_name", "position", "age", "games",
|
||||
"minutes_played",
|
||||
"per", "ts_percent", "three_par", "ftr", // ← use three_par & ftr
|
||||
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
|
||||
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
|
||||
"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
|
||||
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
|
||||
}),
|
||||
}).Create(&stat).Error; err != nil {
|
||||
log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
|
||||
}
|
||||
})
|
||||
// 6) Map into your GORM model.
|
||||
stat := models.PlayerAdvancedStat{
|
||||
ExternalID: extID,
|
||||
PlayerID: playerID,
|
||||
PlayerName: playerName,
|
||||
Position: data["pos"],
|
||||
Age: mustAtoi(data["age"]),
|
||||
Games: g,
|
||||
MinutesPlayed: mustAtoi(data["mp"]),
|
||||
PER: mustParseFloat(data["per"]),
|
||||
TSPercent: mustParseFloat(data["ts_pct"]),
|
||||
ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]),
|
||||
FTR: mustParseFloat(data["fta_per_fga_pct"]),
|
||||
OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
|
||||
DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
|
||||
TotalRBPercent: mustParseFloat(data["trb_pct"]),
|
||||
AssistPercent: mustParseFloat(data["ast_pct"]),
|
||||
StealPercent: mustParseFloat(data["stl_pct"]),
|
||||
BlockPercent: mustParseFloat(data["blk_pct"]),
|
||||
TurnoverPercent: mustParseFloat(data["tov_pct"]),
|
||||
UsagePercent: mustParseFloat(data["usg_pct"]),
|
||||
OffensiveWS: mustParseFloat(data["ows"]),
|
||||
DefensiveWS: mustParseFloat(data["dws"]),
|
||||
WinShares: mustParseFloat(data["ws"]),
|
||||
WinSharesPer: mustParseFloat(data["ws_per_48"]),
|
||||
OffensiveBox: mustParseFloat(data["obpm"]),
|
||||
DefensiveBox: mustParseFloat(data["dbpm"]),
|
||||
Box: mustParseFloat(data["bpm"]),
|
||||
VORP: mustParseFloat(data["vorp"]),
|
||||
Team: teamID,
|
||||
Season: season,
|
||||
IsPlayoff: isPlayoff,
|
||||
}
|
||||
|
||||
return nil
|
||||
// Add the parsed stat object to our slice.
|
||||
statsToUpsert = append(statsToUpsert, stat)
|
||||
})
|
||||
|
||||
// 7) Perform the batch upsert operation after collecting all rows.
|
||||
if len(statsToUpsert) > 0 {
|
||||
log.Printf("Attempting to batch upsert %d advanced player stats for season %d...", len(statsToUpsert), season)
|
||||
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "player_id"},
|
||||
{Name: "season"},
|
||||
{Name: "team"},
|
||||
{Name: "is_playoff"},
|
||||
},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"external_id", "player_name", "position", "age", "games",
|
||||
"minutes_played", "per", "ts_percent", "three_par", "ftr",
|
||||
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
|
||||
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
|
||||
"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
|
||||
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
|
||||
}),
|
||||
}).Create(&statsToUpsert).Error; err != nil {
|
||||
log.Printf("Failed to batch upsert advanced player stats: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("✅ Successfully batch upserted %d advanced records for season %d.", len(statsToUpsert), season)
|
||||
} else {
|
||||
log.Printf("No advanced player data found to import for season %d.", season)
|
||||
}
|
||||
// --- BATCHING LOGIC END ---
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user