shot chart service and models update

This commit is contained in:
Ravi Prasad
2025-05-13 22:10:42 -05:00
parent 5f67c07579
commit cdf5cf4cd2
14 changed files with 3117 additions and 115 deletions
+17
View File
@@ -0,0 +1,17 @@
package services
import "strconv"
// mustAtoi parses s into an int, or returns 0 on error.
func mustAtoi(s string) int {
i, _ := strconv.Atoi(s)
return i
}
// mustParseFloat parses s into a float64, or returns 0.0 on error.
func mustParseFloat(s string) float64 {
f, _ := strconv.ParseFloat(s, 64)
return f
}
@@ -0,0 +1,155 @@
// File: NBA_Go/services/player_advanced_scrape_service.go
package services
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/nprasad2077/NBA_Go/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
advancedURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
)
// urlForAdvSeason picks the correct URL based on isPlayoff.
func urlForAdvSeason(season int, isPlayoff bool) string {
if isPlayoff {
return fmt.Sprintf(advancedPlayoffURLFmt, season)
}
return fmt.Sprintf(advancedURLFmt, season)
}
// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table
// (regular or playoffs) and upserts into the PlayerAdvancedStat model.
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
url := urlForAdvSeason(season, isPlayoff)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
htmlBytes, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
if err != nil {
return err
}
// 📌 pick the right table selector
var table *goquery.Selection
if isPlayoff {
table = doc.Find("#div_advanced_stats table#advanced_stats")
} else {
table = doc.Find("#div_advanced table#advanced")
}
if table.Length() == 0 {
return fmt.Errorf("could not find advanced stats table")
}
// 1. collect the data-stat keys in header order
var headers []string
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
if stat, ok := th.Attr("data-stat"); ok && stat != "" {
headers = append(headers, stat)
}
})
// add our appended-player column
headers = append(headers, "player-additional")
// 2. iterate each row
table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
if tr.HasClass("thead") {
return // skip repeated headers
}
cells := tr.Find("th, td")
data := make(map[string]string, len(headers))
var playerID string
cells.Each(func(i int, cell *goquery.Selection) {
key := headers[i]
data[key] = strings.TrimSpace(cell.Text())
if id, ok := cell.Attr("data-append-csv"); ok {
playerID = id
}
})
if playerID == "" {
return // not a data row
}
data["player-additional"] = playerID
// 3. map into your model
stat := models.PlayerAdvancedStat{
ExternalID: mustAtoi(data["rk"]),
PlayerID: playerID,
PlayerName: data["player"],
Position: data["pos"],
Age: mustAtoi(data["age"]),
Games: mustAtoi(data["g"]),
MinutesPlayed: mustAtoi(data["mp"]),
PER: mustParseFloat(data["per"]),
TSPercent: mustParseFloat(data["ts_pct"]),
ThreePAR: mustParseFloat(data["three_par"]),
FTR: mustParseFloat(data["ftr"]),
OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
TotalRBPercent: mustParseFloat(data["trb_pct"]),
AssistPercent: mustParseFloat(data["ast_pct"]),
StealPercent: mustParseFloat(data["stl_pct"]),
BlockPercent: mustParseFloat(data["blk_pct"]),
TurnoverPercent: mustParseFloat(data["tov_pct"]),
UsagePercent: mustParseFloat(data["usg_pct"]),
OffensiveWS: mustParseFloat(data["ows"]),
DefensiveWS: mustParseFloat(data["dws"]),
WinShares: mustParseFloat(data["ws"]),
WinSharesPer: mustParseFloat(data["ws_per_48"]),
OffensiveBox: mustParseFloat(data["obpm"]),
DefensiveBox: mustParseFloat(data["dbpm"]),
Box: mustParseFloat(data["bpm"]),
VORP: mustParseFloat(data["vorp"]),
Team: data["team_id"],
Season: season,
IsPlayoff: isPlayoff,
}
// 4. upsert
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
{Name: "season"},
{Name: "team"},
{Name: "is_playoff"},
},
DoUpdates: clause.AssignmentColumns([]string{
"external_id", "player_name", "position", "age", "games", "minutes_played",
"per", "ts_percent", "three_par", "ftr",
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
"usage_percent", "offensive_ws", "defensive_ws", "win_shares",
"win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
}),
}).Create(&stat).Error; err != nil {
log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
}
})
return nil
}
@@ -0,0 +1,206 @@
// File: services/player_shot_chart_scrape_service.go
package services
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"golang.org/x/net/html"
"github.com/nprasad2077/NBA_Go/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// FetchAndStoreShotChartScrapedForPlayer scrapes theshot chart pages on
// BasketballReference for one player and upserts every shot into SQLite.
// A *composite* unique key keeps duplicates out:
//
// (player_id, season, date, qtr, time_remaining, top, left)
//
// The model therefore needs a matching unique index (see models package).
func FetchAndStoreShotChartScrapedForPlayer(
db *gorm.DB,
playerID string,
startSeason, endSeason int,
) error {
// loop newest → oldest
for season := startSeason; season >= endSeason; season-- {
url := fmt.Sprintf(
"https://www.basketball-reference.com/players/%s/%s/shooting/%d",
playerID[:1], playerID, season,
)
// ─────────────────────── 1) HTTP GET ────────────────────────
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return fmt.Errorf("request creation error for %d: %w", season, err)
}
req.Header.Set("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
)
resp, err := (&http.Client{}).Do(req)
if err != nil {
return fmt.Errorf("HTTP error for %d: %w", season, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return fmt.Errorf("unexpected status for %d: %s", season, resp.Status)
}
bodyBytes, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return fmt.Errorf("read error for %d: %w", season, err)
}
// ─────────────────── 2) player name (nice to have) ───────────
fullDoc, err := goquery.NewDocumentFromReader(bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("nameparse error for %d: %w", season, err)
}
playerName := fullDoc.Find("#meta span[itemprop='name']").First().Text()
if playerName == "" {
playerName = playerID
}
// ───────────────── 3) commentedout shot chart HTML ──────────
shotHTML := extractCommentedShotChart(bodyBytes)
if shotHTML == "" {
log.Printf("⚠️ no shotchart comment found for %d", season)
continue
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(shotHTML))
if err != nil {
return fmt.Errorf("snippetparse error for %d: %w", season, err)
}
wrapper := doc.Find("div#div_shot-chart div#shot-wrapper")
if wrapper.Length() == 0 {
log.Printf("⚠️ no shotwrapper for %d", season)
continue
}
// ───────────────────── 4) scrape every tooltip ───────────────
var firstErr error // surface the first DB failure after the loop
wrapper.Find("div.tooltip.make, div.tooltip.miss").Each(func(_ int, s *goquery.Selection) {
// position on the court
style, _ := s.Attr("style")
parts := strings.Split(style, ";")
top := parsePx(parts[0])
left := parsePx(parts[1])
// tooltip text
tip, _ := s.Attr("tip")
tipParts := strings.Split(tip, "<br>")
// date, team, opponent
header := tipParts[0] // e.g. "Oct 20, 2021, CHI at DET"
dateSegs := strings.SplitN(header, ", ", 3) // {"Oct 20", "2021", "CHI at DET"}
date := dateSegs[0] + "," + dateSegs[1] // "Oct 20,2021"
var team, opponent string
if len(dateSegs) == 3 {
game := dateSegs[2]
if p := strings.SplitN(game, " at ", 2); len(p) == 2 {
team, opponent = p[0], p[1]
} else if p := strings.SplitN(game, " vs ", 2); len(p) == 2 {
team, opponent = p[0], p[1]
}
}
// quarter & time remaining
qt := strings.SplitN(tipParts[1], ",", 2)
quarter := qt[0]
timeRem := strings.Fields(qt[1])[0]
// result, shot type & distance
rt := strings.Fields(tipParts[2]) // ["Made","2-pointer","..."|"Missed",...]
made := rt[0] == "Made"
shotType := rt[1]
distance := mustAtoi(rt[len(rt)-2])
// score & lead flag
last := strings.Fields(tipParts[3])
sc := strings.Split(last[len(last)-1], "-")
teamScore, oppScore := mustAtoi(sc[0]), mustAtoi(sc[1])
lead := teamScore > oppScore
shot := models.PlayerShotChart{
PlayerID: playerID,
PlayerName: playerName,
Top: top,
Left: left,
Date: date,
Quarter: quarter,
TimeRemaining: timeRem,
Result: made,
ShotType: shotType,
DistanceFt: distance,
Lead: lead,
TeamScore: teamScore,
OpponentTeamScore: oppScore,
Opponent: opponent,
Team: team,
Season: season,
}
// ─────────────── 5) upsert / dedup ────────────────
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{ // MUST match the unique index order
{Name: "player_id"},
{Name: "season"},
{Name: "date"},
{Name: "qtr"},
{Name: "time_remaining"},
{Name: "top"},
{Name: "left"},
},
DoUpdates: clause.AssignmentColumns([]string{
"player_name", "result", "shot_type", "distance_ft",
"lead", "team_score", "opponent_team_score",
"opponent", "team",
}),
}).Create(&shot).Error; err != nil && firstErr == nil {
firstErr = err
}
})
if firstErr != nil {
return fmt.Errorf("DB upsert error for season %d: %w", season, firstErr)
}
}
return nil
}
// extractCommentedShotChart returns the inner HTML of the comment block that
// contains <div id="div_shot-chart" …>. BR hides the SVG there for ad reasons.
func extractCommentedShotChart(htmlBytes []byte) string {
root, err := html.Parse(bytes.NewReader(htmlBytes))
if err != nil {
return ""
}
var found string
var walker func(*html.Node)
walker = func(n *html.Node) {
if n.Type == html.CommentNode && strings.Contains(n.Data, `id="div_shot-chart"`) {
found = n.Data
return
}
for c := n.FirstChild; c != nil && found == ""; c = c.NextSibling {
walker(c)
}
}
walker(root)
return found
}
// parsePx turns "left:244px" or "top:18px" into int(244 / 18).
func parsePx(s string) int {
parts := strings.Split(s, ":")
return mustAtoi(strings.TrimSuffix(parts[1], "px"))
}
+70 -58
View File
@@ -1,76 +1,88 @@
// File: services/player_shot_chart_fetch_service.go
package services
import (
"encoding/json"
"fmt"
"log"
"time"
"encoding/json"
"fmt"
"log"
"time"
"github.com/nprasad2077/NBA_Go/models"
"github.com/nprasad2077/NBA_Go/utils"
"github.com/nprasad2077/NBA_Go/utils/metrics"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/nprasad2077/NBA_Go/models"
"github.com/nprasad2077/NBA_Go/utils"
"github.com/nprasad2077/NBA_Go/utils/metrics"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// FetchAndStoreShotChartForPlayer fetches a single player's shot chart, parses
// each shot's date into a season, and upserts into DB.
func FetchAndStoreShotChartForPlayer(db *gorm.DB, playerId string) error {
metrics.DBOperationsTotal.WithLabelValues("fetch", "player_shot_chart").Inc()
// FetchAndStoreShotChartForPlayer pulls the public NBAAPI JSON for a single
// player, derives the season from the shot date, and upserts into SQLite.
// Dupes are prevented by the unique index:
//
// (player_id, season, date, qtr, time_remaining, top, left)
func FetchAndStoreShotChartForPlayer(db *gorm.DB, playerID string) error {
metrics.DBOperationsTotal.WithLabelValues("fetch", "player_shot_chart").Inc()
url := fmt.Sprintf("http://rest.nbaapi.com/api/ShotChartData/playerid/%s", playerId)
body, err := utils.GetJSON(url)
if err != nil {
return err
}
url := fmt.Sprintf("http://rest.nbaapi.com/api/ShotChartData/playerid/%s", playerID)
body, err := utils.GetJSON(url)
if err != nil {
return err
}
var shots []models.PlayerShotChart
if err := json.Unmarshal(body, &shots); err != nil {
return err
}
var shots []models.PlayerShotChart
if err := json.Unmarshal(body, &shots); err != nil {
return err
}
for _, shot := range shots {
// parse date "Feb 11, 2023" → time.Time
if t, err := time.Parse("Jan 2, 2006", shot.Date); err == nil {
shot.Season = t.Year()
}
for _, shot := range shots {
// "Feb 11, 2023" → season 2023
if t, err := time.Parse("Jan 2, 2006", shot.Date); err == nil {
shot.Season = t.Year()
}
err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "player_id"}, {Name: "external_id"}},
DoUpdates: clause.AssignmentColumns([]string{
"player_name", "top", "left", "date", "qtr",
"time_remaining", "result", "shot_type", "distance_ft",
"lead", "team_score", "opponent_team_score", "opponent",
"team", "season",
}),
}).Create(&shot).Error
err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
{Name: "season"},
{Name: "date"},
{Name: "qtr"},
{Name: "time_remaining"},
{Name: "top"},
{Name: "left"},
},
// update mutable fields; leave the identity columns untouched
DoUpdates: clause.AssignmentColumns([]string{
"player_name", "result", "shot_type", "distance_ft",
"lead", "team_score", "opponent_team_score",
"opponent", "team",
}),
}).Create(&shot).Error
metrics.DBOperationsTotal.WithLabelValues("store", "player_shot_chart").Inc()
metrics.DBOperationsTotal.WithLabelValues("store", "player_shot_chart").Inc()
if err != nil {
log.Printf("Failed to upsert shot chart for %s id=%d: %v",
shot.PlayerID, shot.ExternalID, err)
}
}
return nil
if err != nil {
log.Printf("❌ upsert failed for %s on %s (%s %s): %v",
shot.PlayerID, shot.Date, shot.Quarter, shot.TimeRemaining, err)
}
}
return nil
}
// FetchAndStoreAllPlayerShotCharts loads every distinct playerId from your stats
// tables and invokes the per-player fetch.
// FetchAndStoreAllPlayerShotCharts enumerates every distinct player_id in your
// totals table, then calls the perplayer loader. It sleeps ~1.1s between
// requests to stay well under publicAPI rate limits.
func FetchAndStoreAllPlayerShotCharts(db *gorm.DB) error {
var playerIds []string
var playerIDs []string
// gather from total stats (could also include advanced stats)
db.Model(&models.PlayerTotalStat{}).
Distinct("player_id").
Pluck("player_id", &playerIds)
// collect IDs (add other stats tables if needed)
db.Model(&models.PlayerTotalStat{}).
Distinct("player_id").
Pluck("player_id", &playerIDs)
for _, pid := range playerIds {
if err := FetchAndStoreShotChartForPlayer(db, pid); err != nil {
log.Printf("Error importing shot chart for %s: %v", pid, err)
}
// throttle
time.Sleep(1100 * time.Millisecond)
}
return nil
for _, pid := range playerIDs {
if err := FetchAndStoreShotChartForPlayer(db, pid); err != nil {
log.Printf("Error importing shot chart for %s: %v", pid, err)
}
time.Sleep(1100 * time.Millisecond) // throttle
}
return nil
}
-13
View File
@@ -8,7 +8,6 @@ import (
"io"
"log"
"net/http"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
@@ -156,16 +155,4 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
})
return nil
}
// mustAtoi parses integer or returns 0
func mustAtoi(s string) int {
i, _ := strconv.Atoi(s)
return i
}
// mustParseFloat parses float or returns 0.0
func mustParseFloat(s string) float64 {
f, _ := strconv.ParseFloat(s, 64)
return f
}