nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

per game stats and box score

Browse Source
This commit is contained in:
Ravi Prasad
2025-07-08 01:23:34 -05:00
parent a16af088af
commit c7c04fee7e
7 changed files with 704 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/database.go
+25 -4
View File
@@ -1,3 +1,5 @@
// File: config/database.go
package config
import (
@@ -12,7 +14,7 @@ import (
)
func InitDB(shouldMigrate bool) *gorm.DB {
// CHANGE: Build the DSN from environment variables
// ... (DSN setup code is unchanged) ...
dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC",
os.Getenv("DB_HOST"),
os.Getenv("DB_USER"),
@@ -21,13 +23,32 @@ func InitDB(shouldMigrate bool) *gorm.DB {
os.Getenv("DB_PORT"),
)
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) // <- CHANGE: Use the postgres driver
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
if err != nil {
log.Fatalf("failed to connect database: %v", err)
}
metrics.DBOperationsTotal.WithLabelValues("connect", "database").Inc()
if shouldMigrate {
// --- ADD THIS BLOCK TO DROP STALE TABLES ---
// Drop tables in reverse order of dependency (children first).
// This ensures a clean migration every time the import process runs.
log.Println("⚠️ Dropping existing game-related tables for a clean migration...")
if err := db.Migrator().DropTable(
&models.LineScore{},
&models.PlayerGameBasicStat{},
&models.PlayerGameAdvStat{},
&models.TeamGameBasicStat{},
&models.TeamGameAdvStat{},
&models.Game{}, // Drop parent table last
); err != nil {
log.Fatalf("failed to drop tables: %v", err)
}
log.Println("✅ Tables dropped successfully.")
// --- END OF ADDED BLOCK ---
// Your existing AutoMigrate calls will now work correctly
if err := db.AutoMigrate(&models.PlayerAdvancedStat{}); err != nil {
log.Fatalf("migrate PlayerAdvancedStat: %v", err)
}
@@ -40,7 +61,7 @@ func InitDB(shouldMigrate bool) *gorm.DB {
if err := db.AutoMigrate(&models.APIKey{}); err != nil {
log.Fatalf("migrate APIKey: %v", err)
}
// Game Models
// Game Models will be recreated with the correct schema
if err := db.AutoMigrate(
&models.Game{},
&models.LineScore{},
@@ -55,4 +76,4 @@ func InitDB(shouldMigrate bool) *gorm.DB {
}
return db
}
}
import.go
+47 -4
View File
@@ -11,7 +11,7 @@ import (
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2022; season <= 2025; season++ {
for season := 2025; season <= 2025; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
@@ -23,7 +23,7 @@ func importPlayerAdvanced(db *gorm.DB) {
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2022; season <= 2025; season++ {
for season := 2025; season <= 2025; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
@@ -35,7 +35,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) {
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) {
for season := 2022; season <= 2025; season++ {
for season := 2025; season <= 2025; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2022; season <= 2025; season++ {
for season := 2025; season <= 2025; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
@@ -57,6 +57,49 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
}
}
// importGameSchedules fetches and stores game schedules
func importGameSchedules(db *gorm.DB) {
// An NBA season typically runs from October to June
months := []string{
"october", "november", "december", "january",
"february", "march", "april", "may", "june",
}
for season := 2025; season <= 2025; season++ {
log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
for _, month := range months {
// The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished)
if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil {
// Log the error but continue to the next month/season
log.Printf("Game schedule import failed for %s %d: %v", month, season, err)
}
log.Printf("Game schedule import for %s, %d complete.", month, season)
// Respectful delay between requests
time.Sleep(1100 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
}
}
// importBoxScores fetches and stores all box score data (line scores, player/team stats)
// for games within a recent date range.
func importBoxScores(db *gorm.DB) {
// Define the date range for the import.
// This example fetches data for games in the last 90 days.
to := time.Now()
from := to.AddDate(0, -1, 1) // 0 years, -3 months, 0 days
log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02"))
if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
log.Fatalf("Box score import failed: %v", err)
}
log.Printf("--- Finished Box Score Data Import ---")
}
main.go
+7
View File
@@ -61,6 +61,13 @@ func main() {
importPlayerTotalsPlayoffsScrape(db)
log.Println("🎉 Player Playoffs (scraped) Import completed successfully")
importGameSchedules(db)
log.Println("🎉 Game Imports completed successfully 🏀")
importBoxScores(db)
log.Println("🎉 Related Box-Score Imports completed successfully 📦")
log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
return
}
models/game.go
+91 -93
View File
@@ -1,32 +1,30 @@
package models
import (
"time"
"gorm.io/gorm"
"time"
)
// Game represents the top-level information for a single NBA game, including schedule details and results.
type Game struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex" json:"gameId"` // A unique identifier for the game, e.g., "202504010ATL"
Date time.Time `gorm:"not null;index" json:"date"`
IsPlayoff bool `gorm:"not null;default:false;index" json:"isPlayoff"`
StartTimeET string `json:"startTimeET"`
Arena string `json:"arena"`
VisitorTeam string `gorm:"not null" json:"visitorTeam"`
VisitorPTS int `json:"visitorPts"`
HomeTeam string `gorm:"not null" json:"homeTeam"`
HomePTS int `json:"homePts"`
GameDuration string `json:"gameDuration"` // e.g., "2:23"
BoxScoreURL string `json:"boxScoreUrl"`
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex" json:"gameId"` // The field you want to use
Date time.Time `gorm:"not null;index" json:"date"`
IsPlayoff bool `gorm:"not null;default:false;index" json:"isPlayoff"`
StartTimeET string `json:"startTimeET"`
Arena string `json:"arena"`
VisitorTeam string `gorm:"not null" json:"visitorTeam"`
VisitorPTS int `json:"visitorPts"`
HomeTeam string `gorm:"not null" json:"homeTeam"`
HomePTS int `json:"homePts"`
GameDuration string `json:"gameDuration"`
BoxScoreURL string `json:"boxScoreUrl"`
// Associations
LineScores []LineScore `json:"lineScores"`
PlayerGameBasicStats []PlayerGameBasicStat `json:"playerGameBasicStats"`
PlayerGameAdvStats []PlayerGameAdvStat `json:"playerGameAdvStats"`
TeamGameBasicStats []TeamGameBasicStat `json:"teamGameBasicStats"`
TeamGameAdvStats []TeamGameAdvStat `json:"teamGameAdvStats"`
// Associations (Updated with explicit foreign key tags)
LineScores []LineScore `gorm:"foreignKey:GameID;references:GameID" json:"lineScores"`
PlayerGameBasicStats []PlayerGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameBasicStats"`
PlayerGameAdvStats []PlayerGameAdvStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameAdvStats"`
TeamGameBasicStats []TeamGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"teamGameBasicStats"`
TeamGameAdvStats []TeamGameAdvStat `gorm:"foreignKey:GameID;references:GameID" json:"teamGameAdvStats"`
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
@@ -43,10 +41,10 @@ type LineScore struct {
Q3 int `json:"q3"`
Q4 int `json:"q4"`
// Overtime scores; will be 0 if the game did not go to the respective OT period.
OT1 int `json:"ot1"`
OT2 int `json:"ot2"`
OT3 int `json:"ot3"`
Total int `json:"total"`
OT1 int `json:"ot1"`
OT2 int `json:"ot2"`
OT3 int `json:"ot3"`
Total int `json:"total"`
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
@@ -55,33 +53,33 @@ type LineScore struct {
// PlayerGameBasicStat contains the basic box score statistics for a single player in a single game.
type PlayerGameBasicStat struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"gameId"`
PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"playerId"`
PlayerName string `json:"playerName"`
Team string `gorm:"not null" json:"team"`
Status string `json:"status"` // e.g., "Starter", "Bench", "Did Not Play"
MP string `json:"mp"` // Minutes Played, e.g., "38:07"
FG int `json:"fg"`
FGA int `json:"fga"`
FGPercent float64 `json:"fgPercent"`
ThreeP int `json:"threeP"`
ThreePA int `json:"threePa"`
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"gameId"`
PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"playerId"`
PlayerName string `json:"playerName"`
Team string `gorm:"not null" json:"team"`
Status string `json:"status"` // e.g., "Starter", "Bench", "Did Not Play"
MP string `json:"mp"` // Minutes Played, e.g., "38:07"
FG int `json:"fg"`
FGA int `json:"fga"`
FGPercent float64 `json:"fgPercent"`
ThreeP int `json:"threeP"`
ThreePA int `json:"threePa"`
ThreePPercent float64 `json:"threePPercent"`
FT int `json:"ft"`
FTA int `json:"fta"`
FTPercent float64 `json:"ftPercent"`
ORB int `json:"orb"`
DRB int `json:"drb"`
TRB int `json:"trb"`
AST int `json:"ast"`
STL int `json:"stl"`
BLK int `json:"blk"`
TOV int `json:"tov"`
PF int `json:"pf"`
PTS int `json:"pts"`
GmSc float64 `json:"gmSc"` // Game Score
PlusMinus int `json:"plusMinus"`
FT int `json:"ft"`
FTA int `json:"fta"`
FTPercent float64 `json:"ftPercent"`
ORB int `json:"orb"`
DRB int `json:"drb"`
TRB int `json:"trb"`
AST int `json:"ast"`
STL int `json:"stl"`
BLK int `json:"blk"`
TOV int `json:"tov"`
PF int `json:"pf"`
PTS int `json:"pts"`
GmSc float64 `json:"gmSc"` // Game Score
PlusMinus int `json:"plusMinus"`
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
@@ -90,16 +88,16 @@ type PlayerGameBasicStat struct {
// PlayerGameAdvStat contains the advanced box score statistics for a single player in a single game.
type PlayerGameAdvStat struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"gameId"`
PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"playerId"`
PlayerName string `json:"playerName"`
Team string `gorm:"not null" json:"team"`
MP string `json:"mp"` // Minutes Played
TSPercent float64 `json:"tsPercent"`
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"gameId"`
PlayerID string `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"playerId"`
PlayerName string `json:"playerName"`
Team string `gorm:"not null" json:"team"`
MP string `json:"mp"` // Minutes Played
TSPercent float64 `json:"tsPercent"`
EFGPercent float64 `json:"efgPercent"`
ThreePAr float64 `json:"threePAr"`
FTr float64 `json:"fTr"`
ThreePAr float64 `json:"threePAr"`
FTr float64 `json:"fTr"`
ORBPercent float64 `json:"orbPercent"`
DRBPercent float64 `json:"drbPercent"`
TRBPercent float64 `json:"trbPercent"`
@@ -108,9 +106,9 @@ type PlayerGameAdvStat struct {
BLKPercent float64 `json:"blkPercent"`
TOVPercent float64 `json:"tovPercent"`
USGPercent float64 `json:"usgPercent"`
ORtg int `json:"oRtg"`
DRtg int `json:"dRtg"`
BPM float64 `json:"bpm"` // Box Plus/Minus
ORtg int `json:"oRtg"`
DRtg int `json:"dRtg"`
BPM float64 `json:"bpm"` // Box Plus/Minus
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
@@ -119,28 +117,28 @@ type PlayerGameAdvStat struct {
// TeamGameBasicStat holds the total basic stats for a team in a single game.
type TeamGameBasicStat struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"gameId"`
Team string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"team"`
MP int `json:"mp"` // Total minutes, usually 240 for a regulation game
FG int `json:"fg"`
FGA int `json:"fga"`
FGPercent float64 `json:"fgPercent"`
ThreeP int `json:"threeP"`
ThreePA int `json:"threePa"`
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"gameId"`
Team string `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"team"`
MP int `json:"mp"` // Total minutes, usually 240 for a regulation game
FG int `json:"fg"`
FGA int `json:"fga"`
FGPercent float64 `json:"fgPercent"`
ThreeP int `json:"threeP"`
ThreePA int `json:"threePa"`
ThreePPercent float64 `json:"threePPercent"`
FT int `json:"ft"`
FTA int `json:"fta"`
FTPercent float64 `json:"ftPercent"`
ORB int `json:"orb"`
DRB int `json:"drb"`
TRB int `json:"trb"`
AST int `json:"ast"`
STL int `json:"stl"`
BLK int `json:"blk"`
TOV int `json:"tov"`
PF int `json:"pf"`
PTS int `json:"pts"`
FT int `json:"ft"`
FTA int `json:"fta"`
FTPercent float64 `json:"ftPercent"`
ORB int `json:"orb"`
DRB int `json:"drb"`
TRB int `json:"trb"`
AST int `json:"ast"`
STL int `json:"stl"`
BLK int `json:"blk"`
TOV int `json:"tov"`
PF int `json:"pf"`
PTS int `json:"pts"`
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
@@ -149,14 +147,14 @@ type TeamGameBasicStat struct {
// TeamGameAdvStat holds the total advanced stats for a team in a single game.
type TeamGameAdvStat struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"gameId"`
Team string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"team"`
MP int `json:"mp"`
TSPercent float64 `json:"tsPercent"`
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"gameId"`
Team string `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"team"`
MP int `json:"mp"`
TSPercent float64 `json:"tsPercent"`
EFGPercent float64 `json:"efgPercent"`
ThreePAr float64 `json:"threePAr"`
FTr float64 `json:"fTr"`
ThreePAr float64 `json:"threePAr"`
FTr float64 `json:"fTr"`
ORBPercent float64 `json:"orbPercent"`
DRBPercent float64 `json:"drbPercent"`
TRBPercent float64 `json:"trbPercent"`
@@ -165,8 +163,8 @@ type TeamGameAdvStat struct {
BLKPercent float64 `json:"blkPercent"`
TOVPercent float64 `json:"tovPercent"`
USGPercent float64 `json:"usgPercent"` // Will be ~100.0 for a team
ORtg float64 `json:"oRtg"`
DRtg float64 `json:"dRtg"`
ORtg float64 `json:"oRtg"`
DRtg float64 `json:"dRtg"`
CreatedAt time.Time `swaggerignore:"true"`
UpdatedAt time.Time `swaggerignore:"true"`
services/box_score_scrape_service.go
+334
View File
@@ -0,0 +1,334 @@
// File: services/box_score_scrape_service.go
package services
import (
"fmt"
"log"
"net/http"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/nprasad2077/NBA_Go/models"
"github.com/nprasad2077/NBA_Go/utils"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const boxScoreURLBase = "https://www.basketball-reference.com"
// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
var games []models.Game
// Query the database for games within the specified date range.
if err := db.Where("date >= ? AND date < ?", from, to).Find(&games).Error; err != nil {
return fmt.Errorf("failed to query games from DB: %w", err)
}
log.Printf("Found %d games to process in the specified date range.", len(games))
for _, game := range games {
log.Printf("Processing game: %s", game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil {
// Log the error but continue to the next game
log.Printf("Error processing box score for game %s: %v", game.GameID, err)
}
// Be a good internet citizen and pause between requests.
utils.SleepWithJitter(1500 * time.Millisecond)
}
return nil
}
// scrapeBoxScorePage handles fetching and parsing a single box score page.
func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("received non-200 status code: %s", resp.Status)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return err
}
// The content is often inside a comment, so we need to extract it.
doc = uncommentDoc(doc)
// --- Scrape all data types from the page ---
if err := parseAndStoreLineScore(db, doc, gameID); err != nil {
return err
}
if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
return err
}
return nil
}
// parseAndStoreLineScore scrapes the line score table.
func parseAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
var lineScores []models.LineScore
doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
ls := models.LineScore{GameID: gameID}
ls.Team = row.Find(`th[data-stat="team"] a`).Text()
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
})
if len(lineScores) > 0 {
return db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error
}
return nil
}
// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
var allPlayerBasicStats []models.PlayerGameBasicStat
var allPlayerAdvStats []models.PlayerGameAdvStat
var allTeamBasicStats []models.TeamGameBasicStat
var allTeamAdvStats []models.TeamGameAdvStat
// Use a CSS attribute selector to find all box score tables for both teams.
doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) {
tableID, _ := table.Attr("id")
isAdvanced := strings.Contains(tableID, "-advanced")
teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic")
teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced")
// Process player rows
table.Find("tbody tr").Each(func(j int, row *goquery.Selection) {
playerID, exists := row.Find("th").Attr("data-append-csv")
if !exists || playerID == "" {
return // Not a player row
}
// Handle "Did Not Play" or other statuses
reason := row.Find(`td[data-stat="reason"]`)
status := "Played"
if reason.Length() > 0 {
status = reason.Text()
}
if !isAdvanced {
stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status)
allPlayerBasicStats = append(allPlayerBasicStats, stat)
} else {
stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status)
allPlayerAdvStats = append(allPlayerAdvStats, stat)
}
})
// Process team total row
table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) {
if !isAdvanced {
stat := parseTeamBasicStat(row, gameID, teamAbbr)
allTeamBasicStats = append(allTeamBasicStats, stat)
} else {
stat := parseTeamAdvStat(row, gameID, teamAbbr)
allTeamAdvStats = append(allTeamAdvStats, stat)
}
})
})
// Batch upsert all collected stats
if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil {
return err
}
return nil
}
// --- Parsing Helper Functions ---
func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat {
return models.PlayerGameBasicStat{
GameID: gameID,
PlayerID: playerID,
PlayerName: row.Find(`th[data-stat="player"] a`).Text(),
Team: team,
Status: status,
MP: row.Find(`td[data-stat="mp"]`).Text(),
FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
FGPercent: mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
ThreeP: mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
ThreePA: mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
FT: mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
FTA: mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
FTPercent: mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
ORB: mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
DRB: mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
TRB: mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
AST: mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
STL: mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
BLK: mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
TOV: mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
PF: mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
PTS: mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
GmSc: mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()),
PlusMinus: mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()),
}
}
func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat {
return models.PlayerGameAdvStat{
GameID: gameID,
PlayerID: playerID,
PlayerName: row.Find(`th[data-stat="player"] a`).Text(),
Team: team,
MP: row.Find(`td[data-stat="mp"]`).Text(),
TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
ThreePAr: mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
FTr: mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
ORtg: mustAtoi(row.Find(`td[data-stat="off_rtg"]`).Text()),
DRtg: mustAtoi(row.Find(`td[data-stat="def_rtg"]`).Text()),
BPM: mustParseFloat(row.Find(`td[data-stat="bpm"]`).Text()),
}
}
func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat {
return models.TeamGameBasicStat{
GameID: gameID,
Team: team,
MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
FG: mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
FGA: mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
FGPercent: mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
ThreeP: mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
ThreePA: mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
FT: mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
FTA: mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
FTPercent: mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
ORB: mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
DRB: mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
TRB: mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
AST: mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
STL: mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
BLK: mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
TOV: mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
PF: mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
PTS: mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
}
}
func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat {
return models.TeamGameAdvStat{
GameID: gameID,
Team: team,
MP: mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
TSPercent: mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
ThreePAr: mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
FTr: mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
ORtg: mustParseFloat(row.Find(`td[data-stat="off_rtg"]`).Text()),
DRtg: mustParseFloat(row.Find(`td[data-stat="def_rtg"]`).Text()),
}
}
// --- DB and Utility Functions ---
func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
if len(pbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
}).Create(&pbs).Error; err != nil {
return fmt.Errorf("failed to upsert player basic stats: %w", err)
}
}
if len(pas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
}).Create(&pas).Error; err != nil {
return fmt.Errorf("failed to upsert player advanced stats: %w", err)
}
}
if len(tbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
}).Create(&tbs).Error; err != nil {
return fmt.Errorf("failed to upsert team basic stats: %w", err)
}
}
if len(tas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
}).Create(&tas).Error; err != nil {
return fmt.Errorf("failed to upsert team advanced stats: %w", err)
}
}
return nil
}
// uncommentDoc finds and replaces commented out HTML sections.
func uncommentDoc(doc *goquery.Document) *goquery.Document {
doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) {
s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool {
return goquery.NodeName(s) == "#comment"
}).Each(func(i int, s *goquery.Selection) {
uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text()))
// FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string.
s.ReplaceWithSelection(uncommented.Find("body").Children())
})
})
return doc
}
// getModelColumns is a placeholder for a more robust reflection-based column name generator.
// For now, it returns hardcoded lists.
func getModelColumns(model interface{}) []string {
switch model.(type) {
case *models.PlayerGameBasicStat:
return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"}
case *models.PlayerGameAdvStat:
return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"}
case *models.TeamGameBasicStat:
return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"}
case *models.TeamGameAdvStat:
return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"}
}
return []string{}
}
services/game_scrape_service.go
+190
View File
@@ -0,0 +1,190 @@
// File: services/game_scrape_service.go
package services
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/nprasad2077/NBA_Go/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const gameScheduleURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_games-%s.html"
// FetchAndStoreGameSchedule scrapes the game schedule for a given season and month.
// The month should be the full lowercase name, e.g., "october", "november".
// If db is nil, it will perform a "dry run" and print the parsed data to the console.
func FetchAndStoreGameSchedule(db *gorm.DB, season int, month string) error {
url := fmt.Sprintf(gameScheduleURLFmt, season, month)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("failed to fetch schedule for %s %d: %w", month, season, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Printf("⚠️ Skipping schedule for %s %d (Status: %s)", month, season, resp.Status)
return nil // Not a fatal error, just no data for this month.
}
htmlBytes, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read response body for %s %d: %w", month, season, err)
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
if err != nil {
return fmt.Errorf("failed to parse HTML for %s %d: %w", month, season, err)
}
table := doc.Find("table#schedule")
if table.Length() == 0 {
// Sometimes the content is commented out
commentNode := doc.Find("#all_schedule").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
return goquery.NodeName(s) == "#comment"
})
if commentNode.Length() > 0 {
commentedHTML := commentNode.Nodes[0].FirstChild.Data
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
if err != nil {
return fmt.Errorf("failed to parse commented schedule HTML: %w", err)
}
table = innerDoc.Find("table#schedule")
}
}
if table.Length() == 0 {
log.Printf("No schedule table found for %s %d.", month, season)
return nil
}
var gamesToUpsert []models.Game
table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
// Skip table header rows that are sometimes repeated in the body
if row.Find("th.poptip").Length() > 1 {
return
}
// Skip rows that don't represent games (e.g., placeholder rows)
if row.Find(`[data-stat="visitor_team_name"]`).Text() == "" {
return
}
var game models.Game
var gameID string
// Extract GameID from the box score link, which is the most reliable unique key
boxScoreCell := row.Find(`td[data-stat="box_score_text"] a`)
if href, exists := boxScoreCell.Attr("href"); exists {
parts := strings.Split(href, "/")
fileName := parts[len(parts)-1]
gameID = strings.TrimSuffix(fileName, ".html")
}
// If there's no box score link, it's likely a future game, we can skip it or handle differently
if gameID == "" {
return
}
game.GameID = gameID
// Get the date part from the 'csk' attribute for accuracy
dateCsk, _ := row.Find(`th[data-stat="date_game"]`).Attr("csk")
var datePart string
if len(dateCsk) >= 8 {
datePart = dateCsk[:8]
} else {
log.Printf("Could not parse date from invalid 'csk' attribute: %s. Skipping.", dateCsk)
return
}
// Get the start time string
startTimeET := row.Find(`td[data-stat="game_start_time"]`).Text()
if startTimeET == "" {
// Skip games without a start time, as they can't be parsed accurately
log.Printf("Could not find start time for game %s. Skipping.", gameID)
return
}
// Load the US/Eastern timezone to correctly handle ET/EST/EDT
eastern, err := time.LoadLocation("America/New_York")
if err != nil {
log.Printf("FATAL: Could not load America/New_York timezone: %v", err)
// This is a system-level error, so we stop the row processing here.
// The function will continue and process any games already parsed.
return
}
// Combine date and time and parse together in the correct timezone.
// The layout "3:04p" handles times like "7:30p".
fullDateTimeString := datePart + startTimeET
layout := "200601023:04p"
gameDate, err := time.ParseInLocation(layout, fullDateTimeString, eastern)
if err != nil {
log.Printf("Could not parse combined date-time for game %s (value: '%s'): %v. Skipping.", gameID, fullDateTimeString, err)
return
}
game.Date = gameDate
game.StartTimeET = startTimeET // Keep the original string as well
game.VisitorTeam = row.Find(`td[data-stat="visitor_team_name"] a`).Text()
game.VisitorPTS = mustAtoi(row.Find(`td[data-stat="visitor_pts"]`).Text())
game.HomeTeam = row.Find(`td[data-stat="home_team_name"] a`).Text()
game.HomePTS = mustAtoi(row.Find(`td[data-stat="home_pts"]`).Text())
game.BoxScoreURL, _ = boxScoreCell.Attr("href")
game.GameDuration = row.Find(`td[data-stat="game_duration"]`).Text()
game.Arena = row.Find(`td[data-stat="arena_name"]`).Text()
game.IsPlayoff = strings.Contains(row.Find(`td[data-stat="game_remarks"]`).Text(), "Playoffs")
gamesToUpsert = append(gamesToUpsert, game)
})
if len(gamesToUpsert) > 0 {
// If the db connection is nil, we're in test/debug mode. Print to console.
if db == nil {
log.Println("--- RUNNING IN DRY-RUN MODE ---")
for _, game := range gamesToUpsert {
// Use %+v to print the struct with field names for clarity
log.Printf("Game Data: %+v\n", game)
}
log.Printf("--- WOULD INSERT %d RECORDS ---", len(gamesToUpsert))
return nil // End execution for dry-run
}
log.Printf("Attempting to batch upsert %d games for %s %d...", len(gamesToUpsert), month, season)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}},
DoUpdates: clause.AssignmentColumns(allGameColumns()),
}).Create(&gamesToUpsert).Error; err != nil {
log.Printf("Failed to batch upsert games: %v", err)
return err
}
log.Printf("✅ Successfully batch upserted %d game records for %s %d.", len(gamesToUpsert), month, season)
} else {
log.Printf("No game data found to import for %s %d.", month, season)
}
return nil
}
// allGameColumns returns a list of all column names in the Game model for the upsert operation.
// This ensures that if a record exists, all its fields are updated with the new data.
func allGameColumns() []string {
return []string{
"date", "is_playoff", "start_time_et", "arena", "visitor_team",
"visitor_pts", "home_team", "home_pts", "game_duration", "box_score_url",
"updated_at",
}
}
services/helpers.go
+10
View File
@@ -8,6 +8,16 @@ func mustAtoi(s string) int {
return i
}
// mustAtoiWithSign handles strings that might have a "+" or "-" sign.
func mustAtoiWithSign(s string) int {
if s == "" {
return 0
}
// The strconv.Atoi function handles the sign automatically.
i, _ := strconv.Atoi(s)
return i
}
// mustParseFloat parses s into a float64, or returns 0.0 on error.
func mustParseFloat(s string) float64 {
f, _ := strconv.ParseFloat(s, 64)