boxscore batch import

docker-compose.local.yml

@@ -25,7 +25,7 @@ services:
   db-init:
     build: .
     env_file: .env
-    command: ["/nba_go"]
+    command: ["/nba_go", "import-data"]
     depends_on:
       postgres:
         condition: service_healthy

import.go

@@ -87,20 +87,18 @@ func importGameSchedules(db *gorm.DB) {
 // importBoxScores fetches and stores all box score data (line scores, player/team stats)
 // for games within a recent date range.
 func importBoxScores(db *gorm.DB) {
-	// Define the date range for the import.
+    // Define the date range for the 2023-2024 NBA season.
+    // The regular season typically starts in October and playoffs end in June.
+    from := time.Date(2023, time.October, 1, 0, 0, 0, 0, time.UTC)
+    to := time.Date(2024, time.July, 1, 0, 0, 0, 0, time.UTC)
 
-	// The format is: time.Date(year, month, day, hour, min, sec, nsec, location)
-    // to := time.Date(2019, time.June, 14, 0, 0, 0, 0, time.UTC)
- 	to := time.Now()
-	from := to.AddDate(-1, 0, 0) // 0 years, -3 months, 0 days
+    log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
 
-	log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02"))
+    if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
+        log.Fatalf("Box score import failed: %v", err)
+    }
 
-	if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
-		log.Fatalf("Box score import failed: %v", err)
-	}
-
-	log.Printf("--- Finished Box Score Data Import ---")
+    log.Printf("--- Finished Box Score Data Import ---")
 }
 
 // importPlayerShotChart fetches shot-charts for every known player

services/box_score_scrape_service.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -15,149 +16,229 @@ import (
 )
 
 const boxScoreURLBase = "https://www.basketball-reference.com"
+const numWorkers = 8 // Number of concurrent scrapers. Adjust based on your machine and network.
+
+// ScrapedResult holds all the parsed stats from a single game.
+type ScrapedResult struct {
+	PlayerBasicStats []models.PlayerGameBasicStat
+	PlayerAdvStats   []models.PlayerGameAdvStat
+	TeamBasicStats   []models.TeamGameBasicStat
+	TeamAdvStats     []models.TeamGameAdvStat
+	LineScores       []models.LineScore
+	GameID           string
+	Err              error
+}
 
 // uncommentDoc finds and replaces commented out HTML sections.
 func uncommentDoc(doc *goquery.Document) *goquery.Document {
 	doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
 		return goquery.NodeName(s) == "#comment"
 	}).Each(func(i int, s *goquery.Selection) {
-		// Use .Data on the underlying html.Node to get the comment content.
 		commentText := s.Nodes[0].Data
 		if strings.Contains(commentText, "<table") {
-			// Replace the comment node with its content.
 			s.ReplaceWithHtml(commentText)
 		}
 	})
 	return doc
 }
 
-// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
+// FetchAndStoreBoxScoreDataForDateRange fetches games and batch processes their box scores concurrently.
 func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
 	var games []models.Game
-	// Query the database for games within the specified date range.
-	if err := db.Where("date >= ? AND date < ?", from, to).Find(&games).Error; err != nil {
+	if err := db.Where("date >= ? AND date < ?", from, to.Add(24*time.Hour)).Find(&games).Error; err != nil {
 		return fmt.Errorf("failed to query games from DB: %w", err)
 	}
 
-	log.Printf("Found %d games to process in the specified date range.", len(games))
+	if len(games) == 0 {
+		log.Println("No games found to process in the specified date range.")
+		return nil
+	}
+	log.Printf("Found %d games to process. Initializing concurrent scraping...", len(games))
 
+	// --- Concurrency Setup ---
+	jobs := make(chan models.Game, len(games))
+	results := make(chan ScrapedResult, len(games))
+	var wg sync.WaitGroup
+
+	// Start worker goroutines
+	for w := 1; w <= numWorkers; w++ {
+		wg.Add(1)
+		go scrapeAndParseWorker(w, jobs, results, &wg)
+	}
+
+	// Send jobs to the workers
 	for _, game := range games {
-		log.Printf("Processing game: %s", game.GameID)
-		fullURL := boxScoreURLBase + game.BoxScoreURL
-		if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil {
-			// Log the error but continue to the next game
-			log.Printf("Error processing box score for game %s: %v", game.GameID, err)
+		jobs <- game
+	}
+	close(jobs)
+
+	// Wait for all workers to finish
+	wg.Wait()
+	close(results)
+
+	// --- Aggregation & Final Upsert ---
+	log.Println("All scraping complete. Aggregating results for final batch upsert...")
+	var allPlayerBasicStats []models.PlayerGameBasicStat
+	var allPlayerAdvStats []models.PlayerGameAdvStat
+	var allTeamBasicStats []models.TeamGameBasicStat
+	var allTeamAdvStats []models.TeamGameAdvStat
+	var allLineScores []models.LineScore
+
+	for res := range results {
+		if res.Err != nil {
+			log.Printf("A worker failed on game %s: %v", res.GameID, res.Err)
+			continue
 		}
-		// Be a good internet citizen and pause between requests.
+		allPlayerBasicStats = append(allPlayerBasicStats, res.PlayerBasicStats...)
+		allPlayerAdvStats = append(allPlayerAdvStats, res.PlayerAdvStats...)
+		allTeamBasicStats = append(allTeamBasicStats, res.TeamBasicStats...)
+		allTeamAdvStats = append(allTeamAdvStats, res.TeamAdvStats...)
+		allLineScores = append(allLineScores, res.LineScores...)
+	}
+
+	// Upsert Line Scores first
+	if len(allLineScores) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.LineScore{})),
+		}).Create(&allLineScores).Error; err != nil {
+			return fmt.Errorf("failed to upsert line scores: %w", err)
+		}
+		log.Printf("Successfully upserted %d line scores.", len(allLineScores))
+	}
+
+	// Call the batch upsert function with the fully aggregated data
+	if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil {
+		return fmt.Errorf("final batch upsert failed: %w", err)
+	}
+
+	log.Printf("Successfully upserted all box score data for %d games.", len(games))
+	return nil
+}
+
+// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
+func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for game := range jobs {
+		log.Printf("Worker %d: Processing game %s", id, game.GameID)
+		fullURL := boxScoreURLBase + game.BoxScoreURL
+
 		utils.SleepWithJitter(2300 * time.Millisecond)
+
+		req, err := http.NewRequest("GET", fullURL, nil)
+		if err != nil {
+			results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("failed to create request: %w", err)}
+			continue
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("request failed: %w", err)}
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("received non-200 status code: %s", resp.Status)}
+			continue
+		}
+
+		doc, err := goquery.NewDocumentFromReader(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			results <- ScrapedResult{GameID: game.GameID, Err: fmt.Errorf("failed to parse document: %w", err)}
+			continue
+		}
+
+		doc = uncommentDoc(doc)
+
+		lineScores := parseLineScore(doc, game.GameID)
+		pbs, pas, tbs, tas := parseBoxScores(doc, game.GameID)
+
+		results <- ScrapedResult{
+			PlayerBasicStats: pbs,
+			PlayerAdvStats:   pas,
+			TeamBasicStats:   tbs,
+			TeamAdvStats:     tas,
+			LineScores:       lineScores,
+			GameID:           game.GameID,
+			Err:              nil,
+		}
 	}
-	return nil
 }
 
-// scrapeBoxScorePage handles fetching and parsing a single box score page.
-func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("received non-200 status code: %s", resp.Status)
-	}
-
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	// 1. Uncomment all tables in the document first.
-	doc = uncommentDoc(doc)
-
-	// 2. Call the dedicated service to handle line scores.
-	if err := FetchAndStoreLineScore(db, doc, gameID); err != nil {
-		log.Printf("Error processing line score for game %s: %v", gameID, err)
-	}
-
-	// 3. The existing box score parser will now work because its tables are visible.
-	if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
-func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
+// parseBoxScores now returns the slices instead of calling the DB.
+func parseBoxScores(doc *goquery.Document, gameID string) ([]models.PlayerGameBasicStat, []models.PlayerGameAdvStat, []models.TeamGameBasicStat, []models.TeamGameAdvStat) {
 	var allPlayerBasicStats []models.PlayerGameBasicStat
 	var allPlayerAdvStats []models.PlayerGameAdvStat
 	var allTeamBasicStats []models.TeamGameBasicStat
 	var allTeamAdvStats []models.TeamGameAdvStat
 
-	// Use a CSS attribute selector to find all box score tables for both teams.
 	doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) {
 		tableID, _ := table.Attr("id")
 		isAdvanced := strings.Contains(tableID, "-advanced")
+
 		teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic")
 		teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced")
 
-		// Process player rows
 		table.Find("tbody tr").Each(func(j int, row *goquery.Selection) {
 			playerID, exists := row.Find("th").Attr("data-append-csv")
 			if !exists || playerID == "" {
-				return // Not a player row
+				return
 			}
 
-			// Handle "Did Not Play" or other statuses
-			reason := row.Find(`td[data-stat="reason"]`)
 			status := "Played"
-			if reason.Length() > 0 {
+			if reason := row.Find(`td[data-stat="reason"]`); reason.Length() > 0 {
 				status = reason.Text()
 			}
 
 			if !isAdvanced {
-				stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status)
-				allPlayerBasicStats = append(allPlayerBasicStats, stat)
+				allPlayerBasicStats = append(allPlayerBasicStats, parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status))
 			} else {
-				stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status)
-				allPlayerAdvStats = append(allPlayerAdvStats, stat)
+				allPlayerAdvStats = append(allPlayerAdvStats, parsePlayerAdvStat(row, gameID, playerID, teamAbbr))
 			}
 		})
 
-		// Process team total row
 		table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) {
 			if !isAdvanced {
-				stat := parseTeamBasicStat(row, gameID, teamAbbr)
-				allTeamBasicStats = append(allTeamBasicStats, stat)
+				allTeamBasicStats = append(allTeamBasicStats, parseTeamBasicStat(row, gameID, teamAbbr))
 			} else {
-				stat := parseTeamAdvStat(row, gameID, teamAbbr)
-				allTeamAdvStats = append(allTeamAdvStats, stat)
+				allTeamAdvStats = append(allTeamAdvStats, parseTeamAdvStat(row, gameID, teamAbbr))
 			}
 		})
 	})
 
-	// Batch upsert all collected stats
-	if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil {
-		return err
-	}
-
-	return nil
+	return allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats
 }
 
-// --- Parsing Helper Functions ---
+func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
+	var lineScores []models.LineScore
+	doc.Find("#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
+		teamAbbr := row.Find(`th a`).Text()
+		if teamAbbr == "" {
+			return
+		}
+
+		lineScores = append(lineScores, models.LineScore{
+			GameID: gameID,
+			Team:   teamAbbr,
+			Q1:     mustAtoi(row.Find(`td[data-stat="1"]`).Text()),
+			Q2:     mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
+			Q3:     mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
+			Q4:     mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
+			OT1:    mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
+			OT2:    mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
+			OT3:    mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
+			Total:  mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
+		})
+	})
+	return lineScores
+}
 
 func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat {
 	return models.PlayerGameBasicStat{
-		GameID:        gameID,
-		PlayerID:      playerID,
+		GameID:        gameID, PlayerID: playerID, Team: team, Status: status,
 		PlayerName:    row.Find(`th[data-stat="player"] a`).Text(),
-		Team:          team,
-		Status:        status,
 		MP:            row.Find(`td[data-stat="mp"]`).Text(),
 		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
 		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
@@ -178,16 +259,14 @@ func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status
 		PF:            mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
 		PTS:           mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
 		GmSc:          mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()),
-		PlusMinus:     mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()),
+		PlusMinus:     mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()), // <-- UPDATED LINE
 	}
 }
 
-func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat {
+func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team string) models.PlayerGameAdvStat {
 	return models.PlayerGameAdvStat{
-		GameID:     gameID,
-		PlayerID:   playerID,
+		GameID: gameID, PlayerID: playerID, Team: team,
 		PlayerName: row.Find(`th[data-stat="player"] a`).Text(),
-		Team:       team,
 		MP:         row.Find(`td[data-stat="mp"]`).Text(),
 		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
 		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
@@ -209,8 +288,7 @@ func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status s
 
 func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat {
 	return models.TeamGameBasicStat{
-		GameID:        gameID,
-		Team:          team,
+		GameID: gameID, Team: team,
 		MP:            mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
 		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
 		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
@@ -235,8 +313,7 @@ func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.Team
 
 func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat {
 	return models.TeamGameAdvStat{
-		GameID:     gameID,
-		Team:       team,
+		GameID: gameID, Team: team,
 		MP:         mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
 		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
 		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
@@ -255,8 +332,6 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
 	}
 }
 
-// --- DB and Utility Functions ---
-
 func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
 	if len(pbs) > 0 {
 		if err := db.Clauses(clause.OnConflict{
@@ -291,22 +366,4 @@ func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.
 		}
 	}
 	return nil
-}
-
-
-
-// getModelColumns is a placeholder for a more robust reflection-based column name generator.
-// For now, it returns hardcoded lists.
-func getModelColumns(model interface{}) []string {
-	switch model.(type) {
-	case *models.PlayerGameBasicStat:
-		return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"}
-	case *models.PlayerGameAdvStat:
-		return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"}
-	case *models.TeamGameBasicStat:
-		return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"}
-	case *models.TeamGameAdvStat:
-		return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"}
-	}
-	return []string{}
-}
+}

services/helpers.go

@@ -1,27 +1,49 @@
 package services
 
-import "strconv"
+import (
+	"log"
+	"strconv"
+	"sync"
+
+	"gorm.io/gorm/schema"
+)
 
 // mustAtoi parses s into an int, or returns 0 on error.
+// This function remains unchanged for general use.
 func mustAtoi(s string) int {
-    i, _ := strconv.Atoi(s)
-    return i
+	i, _ := strconv.Atoi(s)
+	return i
 }
 
-// mustAtoiWithSign handles strings that might have a "+" or "-" sign.
+// mustAtoiWithSign is the new function to handle strings that might
+// have a "+" or "-" sign, like the 'plus_minus' stat.
 func mustAtoiWithSign(s string) int {
-	if s == "" {
-		return 0
-	}
-	// The strconv.Atoi function handles the sign automatically.
+	// strconv.Atoi already handles signs correctly. This function
+	// provides a clear, semantic name for its specific purpose.
 	i, _ := strconv.Atoi(s)
 	return i
 }
 
 // mustParseFloat parses s into a float64, or returns 0.0 on error.
 func mustParseFloat(s string) float64 {
-    f, _ := strconv.ParseFloat(s, 64)
-    return f
+	f, _ := strconv.ParseFloat(s, 64)
+	return f
 }
 
+// getModelColumns uses reflection to discover model columns for dynamic upserts.
+func getModelColumns(instance interface{}) []string {
+	s, err := schema.Parse(instance, &sync.Map{}, schema.NamingStrategy{})
+	if err != nil {
+		log.Printf("Failed to parse GORM schema: %v", err)
+		return []string{}
+	}
 
+	columns := make([]string, 0, len(s.Fields))
+	for _, field := range s.Fields {
+		if field.PrimaryKey {
+			continue // Skip primary key columns
+		}
+		columns = append(columns, field.DBName)
+	}
+	return columns
+}