nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

box score scrape service updates batch handle

Browse Source
This commit is contained in:
Ravi Prasad
2025-08-21 00:41:36 -05:00
parent 6d7e37bf78
commit 7ca427b386
6 changed files with 113 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.DS_Store
Vendored
BIN
View File
Binary file not shown.
Dockerfile
+2 -2
View File
@@ -4,8 +4,8 @@ FROM golang:1.24-bullseye AS builder
ENV CGO_ENABLED=1
ENV GOOS=linux
# Changed from Apple Silicon arm64 config.
ENV GOARCH=amd64
# ENV GOARCH=arm64
# ENV GOARCH=amd64
ENV GOARCH=arm64
WORKDIR /app
import.go
+36 -31
View File
@@ -3,6 +3,7 @@ package main
import (
"log"
"time"
"fmt"
"github.com/nprasad2077/NBA_Go/services"
"github.com/nprasad2077/NBA_Go/utils"
@@ -11,48 +12,48 @@ import (
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2024; season <= 2025; season++ {
for season := 2003; season <= 2017; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
}
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2024; season <= 2025; season++ {
for season := 2003; season <= 2017; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced Playoffs import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1500 * time.Millisecond)
}
}
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) {
for season := 2024; season <= 2025; season++ {
for season := 2003; season <= 2017; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
log.Printf("Player Totals import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1250 * time.Millisecond)
}
}
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2024; season <= 2025; season++ {
for season := 2003; season <= 2017; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
log.Printf("Player Playoffs Totals import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1700 * time.Millisecond)
}
}
@@ -67,7 +68,7 @@ func importGameSchedules(db *gorm.DB) {
// "february", "march", "april", "may", "june",
}
for season := 2025; season <= 2025; season++ {
for season := 2003; season <= 2008; season++ {
log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
for _, month := range months {
// The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished)
@@ -77,7 +78,7 @@ func importGameSchedules(db *gorm.DB) {
}
log.Printf("Game schedule import for %s, %d complete.", month, season)
// Respectful delay between requests
time.Sleep(1100 * time.Millisecond)
time.Sleep(1400 * time.Millisecond)
utils.SleepWithJitter(1800 * time.Millisecond)
}
log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
@@ -87,20 +88,24 @@ func importGameSchedules(db *gorm.DB) {
// importBoxScores fetches and stores all box score data (line scores, player/team stats)
// for games within a recent date range.
func importBoxScores(db *gorm.DB) {
// Define the date range for the 2023-2024 NBA season.
// The regular season typically starts in October and playoffs end in June.
from := time.Date(2022, time.October, 1, 0, 0, 0, 0, time.UTC)
to := time.Date(2023, time.July, 1, 0, 0, 0, 0, time.UTC)
from := time.Date(2025, time.February, 1, 0, 0, 0, 0, time.UTC)
to := time.Date(2025, time.February, 14, 0, 0, 0, 0, time.UTC)
log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---",
from.Format("January 2, 2006"),
to.Format("January 2, 2006"))
log.Println(dateRangeComment)
if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
log.Fatalf("Box score import failed: %v", err)
}
log.Printf("--- Finished Box Score Data Import ---")
log.Println("--- Finished Box Score Data Import ---")
}
// importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
func importPlayerShotCharts(db *gorm.DB) {
log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -117,21 +122,21 @@ func importPlayerShotCharts(db *gorm.DB) {
// Nikola Jokić: "jokicni01"
// Luka Dončić: "doncilu01"
targetPlayerIDs := []string{
"jamesle01",
"curryst01",
"jokicni01",
"doncilu01",
"hardeja01",
"irvinky01",
"duranke01",
// "youngtr01",
// "lillada01",
// "gilgesh01",
// "brunsja01",
// "edwaran01",
// "mitchdo01",
// "bookede01",
// "derozde01",
// "jamesle01",
// "curryst01",
// "jokicni01",
// "doncilu01",
// "hardeja01",
// "irvinky01",
// "duranke01",
"youngtr01",
"lillada01",
"gilgesh01",
"brunsja01",
"edwaran01",
"mitchdo01",
"bookede01",
"derozde01",
// Add more player IDs here as needed
}
main.go
+5 -6
View File
@@ -39,9 +39,9 @@ import (
"github.com/nprasad2077/NBA_Go/config"
"github.com/nprasad2077/NBA_Go/controllers"
_ "github.com/nprasad2077/NBA_Go/docs"
"github.com/nprasad2077/NBA_Go/routes"
"github.com/nprasad2077/NBA_Go/utils/middleware"
_ "github.com/nprasad2077/NBA_Go/docs"
)
func main() {
@@ -65,12 +65,11 @@ func main() {
// importGameSchedules(db)
// log.Println("🎉 Game Imports completed successfully 🏀")
// importBoxScores(db)
// log.Println("🎉 Related Box Score Imports completed successfully 📦")
importPlayerShotCharts(db)
log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
importBoxScores(db)
log.Println("🎉 Related Box Score Imports completed successfully 📦")
// importPlayerShotCharts(db)
// log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
return
services/box_score_scrape_service.go
+67 -22
View File
@@ -16,7 +16,8 @@ import (
)
const boxScoreURLBase = "https://www.basketball-reference.com"
const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network.
const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 3500 * time.Millisecond
// ScrapedResult holds all the parsed stats from a single game.
type ScrapedResult struct {
@@ -123,7 +124,8 @@ func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- Scrape
log.Printf("Worker %d: Processing game %s", id, game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
utils.SleepWithJitter(2300 * time.Millisecond)
utils.SleepWithJitter(baseDelay)
time.Sleep(1500 * time.Millisecond)
req, err := http.NewRequest("GET", fullURL, nil)
if err != nil {
@@ -332,38 +334,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
}
}
func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
const batchSize = 500 // 💡 A safe chunk size to stay under the limit.
// Batch upsert Player Basic Stats
if len(pbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
}).Create(&pbs).Error; err != nil {
return fmt.Errorf("failed to upsert player basic stats: %w", err)
for i := 0; i < len(pbs); i += batchSize {
end := i + batchSize
if end > len(pbs) {
end = len(pbs)
}
batch := pbs[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert player basic stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d player basic stats.", len(pbs))
}
// Batch upsert Player Advanced Stats
if len(pas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
}).Create(&pas).Error; err != nil {
return fmt.Errorf("failed to upsert player advanced stats: %w", err)
for i := 0; i < len(pas); i += batchSize {
end := i + batchSize
if end > len(pas) {
end = len(pas)
}
batch := pas[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert player advanced stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d player advanced stats.", len(pas))
}
// Batch upsert Team Basic Stats
if len(tbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
}).Create(&tbs).Error; err != nil {
return fmt.Errorf("failed to upsert team basic stats: %w", err)
for i := 0; i < len(tbs); i += batchSize {
end := i + batchSize
if end > len(tbs) {
end = len(tbs)
}
batch := tbs[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert team basic stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d team basic stats.", len(tbs))
}
// Batch upsert Team Advanced Stats
if len(tas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
}).Create(&tas).Error; err != nil {
return fmt.Errorf("failed to upsert team advanced stats: %w", err)
for i := 0; i < len(tas); i += batchSize {
end := i + batchSize
if end > len(tas) {
end = len(tas)
}
batch := tas[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert team advanced stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d team advanced stats.", len(tas))
}
return nil
}