per game stats and box score

config/database.go

@@ -1,3 +1,5 @@
+// File: config/database.go
+
 package config
 
 import (
@@ -12,7 +14,7 @@ import (
 )
 
 func InitDB(shouldMigrate bool) *gorm.DB {
-	// CHANGE: Build the DSN from environment variables
+	// ... (DSN setup code is unchanged) ...
 	dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC",
 		os.Getenv("DB_HOST"),
 		os.Getenv("DB_USER"),
@@ -21,13 +23,32 @@ func InitDB(shouldMigrate bool) *gorm.DB {
 		os.Getenv("DB_PORT"),
 	)
 
-	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) // <- CHANGE: Use the postgres driver
+	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
 	if err != nil {
 		log.Fatalf("failed to connect database: %v", err)
 	}
 	metrics.DBOperationsTotal.WithLabelValues("connect", "database").Inc()
 
 	if shouldMigrate {
+		// --- ADD THIS BLOCK TO DROP STALE TABLES ---
+		// Drop tables in reverse order of dependency (children first).
+		// This ensures a clean migration every time the import process runs.
+		log.Println("⚠️ Dropping existing game-related tables for a clean migration...")
+		if err := db.Migrator().DropTable(
+			&models.LineScore{},
+			&models.PlayerGameBasicStat{},
+			&models.PlayerGameAdvStat{},
+			&models.TeamGameBasicStat{},
+			&models.TeamGameAdvStat{},
+			&models.Game{}, // Drop parent table last
+		); err != nil {
+			log.Fatalf("failed to drop tables: %v", err)
+		}
+		log.Println("✅ Tables dropped successfully.")
+		// --- END OF ADDED BLOCK ---
+
+
+		// Your existing AutoMigrate calls will now work correctly
 		if err := db.AutoMigrate(&models.PlayerAdvancedStat{}); err != nil {
 			log.Fatalf("migrate PlayerAdvancedStat: %v", err)
 		}
@@ -40,7 +61,7 @@ func InitDB(shouldMigrate bool) *gorm.DB {
 		if err := db.AutoMigrate(&models.APIKey{}); err != nil {
 			log.Fatalf("migrate APIKey: %v", err)
 		}
-		// Game Models
+		// Game Models will be recreated with the correct schema
 		if err := db.AutoMigrate(
 			&models.Game{},
 			&models.LineScore{},

import.go

@@ -11,7 +11,7 @@ import (
 
 // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
 func importPlayerAdvanced(db *gorm.DB) {
-	for season := 2022; season <= 2025; season++ {
+	for season := 2025; season <= 2025; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
@@ -23,7 +23,7 @@ func importPlayerAdvanced(db *gorm.DB) {
 
 // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
 func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 2022; season <= 2025; season++ {
+	for season := 2025; season <= 2025; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
@@ -35,7 +35,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) {
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-    for season := 2022; season <= 2025; season++ {
+    for season := 2025; season <= 2025; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
             log.Printf("scraped totals import failed for %d: %v", season, err)
         }
@@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-    for season := 2022; season <= 2025; season++ {
+    for season := 2025; season <= 2025; season++ {
         if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
             log.Printf("scraped playoffs import failed for %d: %v", season, err)
         }
@@ -57,6 +57,49 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
     }
 }
 
+// importGameSchedules fetches and stores game schedules
+func importGameSchedules(db *gorm.DB) {
+	// An NBA season typically runs from October to June
+	months := []string{
+		"october", "november", "december", "january",
+		"february", "march", "april", "may", "june",
+	}
+
+	for season := 2025; season <= 2025; season++ {
+		log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
+		for _, month := range months {
+			// The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished)
+			if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil {
+				// Log the error but continue to the next month/season
+				log.Printf("Game schedule import failed for %s %d: %v", month, season, err)
+			}
+			log.Printf("Game schedule import for %s, %d complete.", month, season)
+			// Respectful delay between requests
+			time.Sleep(1100 * time.Millisecond)
+			utils.SleepWithJitter(1000 * time.Millisecond)
+		}
+		log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
+	}
+}
+
+
+// importBoxScores fetches and stores all box score data (line scores, player/team stats)
+// for games within a recent date range.
+func importBoxScores(db *gorm.DB) {
+	// Define the date range for the import.
+	// This example fetches data for games in the last 90 days.
+	to := time.Now()
+	from := to.AddDate(0, -1, 1) // 0 years, -3 months, 0 days
+
+	log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02"))
+
+	if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
+		log.Fatalf("Box score import failed: %v", err)
+	}
+
+	log.Printf("--- Finished Box Score Data Import ---")
+}
+
 
 
 

main.go

@@ -61,6 +61,13 @@ func main() {
 		importPlayerTotalsPlayoffsScrape(db)
 		log.Println("🎉 Player Playoffs (scraped) Import completed successfully")
 
+		importGameSchedules(db)
+		log.Println("🎉 Game Imports completed successfully 🏀")
+
+		importBoxScores(db)
+		log.Println("🎉 Related Box-Score Imports completed successfully 📦")
+
+
 		log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
 		return
 	}

models/game.go

@@ -1,15 +1,13 @@
 package models
 
 import (
-	"time"
-
 	"gorm.io/gorm"
+	"time"
 )
 
-// Game represents the top-level information for a single NBA game, including schedule details and results.
 type Game struct {
 	ID           uint      `gorm:"primaryKey" swaggerignore:"true"`
-	GameID         string    `gorm:"not null;uniqueIndex" json:"gameId"` // A unique identifier for the game, e.g., "202504010ATL"
+	GameID       string    `gorm:"not null;uniqueIndex" json:"gameId"` // The field you want to use
 	Date         time.Time `gorm:"not null;index" json:"date"`
 	IsPlayoff    bool      `gorm:"not null;default:false;index" json:"isPlayoff"`
 	StartTimeET  string    `json:"startTimeET"`
@@ -18,15 +16,15 @@ type Game struct {
 	VisitorPTS   int       `json:"visitorPts"`
 	HomeTeam     string    `gorm:"not null" json:"homeTeam"`
 	HomePTS      int       `json:"homePts"`
-	GameDuration   string    `json:"gameDuration"` // e.g., "2:23"
+	GameDuration string    `json:"gameDuration"`
 	BoxScoreURL  string    `json:"boxScoreUrl"`
 
-	// Associations
-	LineScores           []LineScore           `json:"lineScores"`
-	PlayerGameBasicStats []PlayerGameBasicStat `json:"playerGameBasicStats"`
-	PlayerGameAdvStats   []PlayerGameAdvStat   `json:"playerGameAdvStats"`
-	TeamGameBasicStats   []TeamGameBasicStat   `json:"teamGameBasicStats"`
-	TeamGameAdvStats     []TeamGameAdvStat     `json:"teamGameAdvStats"`
+	// Associations (Updated with explicit foreign key tags)
+	LineScores           []LineScore           `gorm:"foreignKey:GameID;references:GameID" json:"lineScores"`
+	PlayerGameBasicStats []PlayerGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameBasicStats"`
+	PlayerGameAdvStats   []PlayerGameAdvStat   `gorm:"foreignKey:GameID;references:GameID" json:"playerGameAdvStats"`
+	TeamGameBasicStats   []TeamGameBasicStat   `gorm:"foreignKey:GameID;references:GameID" json:"teamGameBasicStats"`
+	TeamGameAdvStats     []TeamGameAdvStat     `gorm:"foreignKey:GameID;references:GameID" json:"teamGameAdvStats"`
 
 	CreatedAt time.Time      `swaggerignore:"true"`
 	UpdatedAt time.Time      `swaggerignore:"true"`

services/box_score_scrape_service.go

@@ -0,0 +1,334 @@
+// File: services/box_score_scrape_service.go
+package services
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"github.com/nprasad2077/NBA_Go/utils"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const boxScoreURLBase = "https://www.basketball-reference.com"
+
+// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
+func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
+	var games []models.Game
+	// Query the database for games within the specified date range.
+	if err := db.Where("date >= ? AND date < ?", from, to).Find(&games).Error; err != nil {
+		return fmt.Errorf("failed to query games from DB: %w", err)
+	}
+
+	log.Printf("Found %d games to process in the specified date range.", len(games))
+
+	for _, game := range games {
+		log.Printf("Processing game: %s", game.GameID)
+		fullURL := boxScoreURLBase + game.BoxScoreURL
+		if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil {
+			// Log the error but continue to the next game
+			log.Printf("Error processing box score for game %s: %v", game.GameID, err)
+		}
+		// Be a good internet citizen and pause between requests.
+		utils.SleepWithJitter(1500 * time.Millisecond)
+	}
+	return nil
+}
+
+// scrapeBoxScorePage handles fetching and parsing a single box score page.
+func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("received non-200 status code: %s", resp.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// The content is often inside a comment, so we need to extract it.
+	doc = uncommentDoc(doc)
+
+	// --- Scrape all data types from the page ---
+	if err := parseAndStoreLineScore(db, doc, gameID); err != nil {
+		return err
+	}
+	if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// parseAndStoreLineScore scrapes the line score table.
+func parseAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
+	var lineScores []models.LineScore
+	doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
+		ls := models.LineScore{GameID: gameID}
+		ls.Team = row.Find(`th[data-stat="team"] a`).Text()
+		ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
+		ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
+		ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
+		ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
+		ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
+		ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
+		ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
+		ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+		lineScores = append(lineScores, ls)
+	})
+
+	if len(lineScores) > 0 {
+		return db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+		}).Create(&lineScores).Error
+	}
+	return nil
+}
+
+// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
+func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
+	var allPlayerBasicStats []models.PlayerGameBasicStat
+	var allPlayerAdvStats []models.PlayerGameAdvStat
+	var allTeamBasicStats []models.TeamGameBasicStat
+	var allTeamAdvStats []models.TeamGameAdvStat
+
+	// Use a CSS attribute selector to find all box score tables for both teams.
+	doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) {
+		tableID, _ := table.Attr("id")
+		isAdvanced := strings.Contains(tableID, "-advanced")
+		teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic")
+		teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced")
+
+		// Process player rows
+		table.Find("tbody tr").Each(func(j int, row *goquery.Selection) {
+			playerID, exists := row.Find("th").Attr("data-append-csv")
+			if !exists || playerID == "" {
+				return // Not a player row
+			}
+
+			// Handle "Did Not Play" or other statuses
+			reason := row.Find(`td[data-stat="reason"]`)
+			status := "Played"
+			if reason.Length() > 0 {
+				status = reason.Text()
+			}
+
+			if !isAdvanced {
+				stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status)
+				allPlayerBasicStats = append(allPlayerBasicStats, stat)
+			} else {
+				stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status)
+				allPlayerAdvStats = append(allPlayerAdvStats, stat)
+			}
+		})
+
+		// Process team total row
+		table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) {
+			if !isAdvanced {
+				stat := parseTeamBasicStat(row, gameID, teamAbbr)
+				allTeamBasicStats = append(allTeamBasicStats, stat)
+			} else {
+				stat := parseTeamAdvStat(row, gameID, teamAbbr)
+				allTeamAdvStats = append(allTeamAdvStats, stat)
+			}
+		})
+	})
+
+	// Batch upsert all collected stats
+	if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// --- Parsing Helper Functions ---
+
+func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat {
+	return models.PlayerGameBasicStat{
+		GameID:        gameID,
+		PlayerID:      playerID,
+		PlayerName:    row.Find(`th[data-stat="player"] a`).Text(),
+		Team:          team,
+		Status:        status,
+		MP:            row.Find(`td[data-stat="mp"]`).Text(),
+		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
+		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
+		FGPercent:     mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
+		ThreeP:        mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
+		ThreePA:       mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
+		ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
+		FT:            mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
+		FTA:           mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
+		FTPercent:     mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
+		ORB:           mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
+		DRB:           mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
+		TRB:           mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
+		AST:           mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
+		STL:           mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
+		BLK:           mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
+		TOV:           mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
+		PF:            mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
+		PTS:           mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
+		GmSc:          mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()),
+		PlusMinus:     mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()),
+	}
+}
+
+func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat {
+	return models.PlayerGameAdvStat{
+		GameID:     gameID,
+		PlayerID:   playerID,
+		PlayerName: row.Find(`th[data-stat="player"] a`).Text(),
+		Team:       team,
+		MP:         row.Find(`td[data-stat="mp"]`).Text(),
+		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
+		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
+		ThreePAr:   mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
+		FTr:        mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
+		ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
+		DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
+		TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
+		ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
+		STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
+		BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
+		TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
+		USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
+		ORtg:       mustAtoi(row.Find(`td[data-stat="off_rtg"]`).Text()),
+		DRtg:       mustAtoi(row.Find(`td[data-stat="def_rtg"]`).Text()),
+		BPM:        mustParseFloat(row.Find(`td[data-stat="bpm"]`).Text()),
+	}
+}
+
+func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat {
+	return models.TeamGameBasicStat{
+		GameID:        gameID,
+		Team:          team,
+		MP:            mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
+		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
+		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
+		FGPercent:     mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
+		ThreeP:        mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
+		ThreePA:       mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
+		ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
+		FT:            mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
+		FTA:           mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
+		FTPercent:     mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
+		ORB:           mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
+		DRB:           mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
+		TRB:           mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
+		AST:           mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
+		STL:           mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
+		BLK:           mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
+		TOV:           mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
+		PF:            mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
+		PTS:           mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
+	}
+}
+
+func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat {
+	return models.TeamGameAdvStat{
+		GameID:     gameID,
+		Team:       team,
+		MP:         mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
+		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
+		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
+		ThreePAr:   mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
+		FTr:        mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
+		ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
+		DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
+		TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
+		ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
+		STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
+		BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
+		TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
+		USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
+		ORtg:       mustParseFloat(row.Find(`td[data-stat="off_rtg"]`).Text()),
+		DRtg:       mustParseFloat(row.Find(`td[data-stat="def_rtg"]`).Text()),
+	}
+}
+
+// --- DB and Utility Functions ---
+
+func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
+	if len(pbs) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
+		}).Create(&pbs).Error; err != nil {
+			return fmt.Errorf("failed to upsert player basic stats: %w", err)
+		}
+	}
+	if len(pas) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
+		}).Create(&pas).Error; err != nil {
+			return fmt.Errorf("failed to upsert player advanced stats: %w", err)
+		}
+	}
+	if len(tbs) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
+		}).Create(&tbs).Error; err != nil {
+			return fmt.Errorf("failed to upsert team basic stats: %w", err)
+		}
+	}
+	if len(tas) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
+		}).Create(&tas).Error; err != nil {
+			return fmt.Errorf("failed to upsert team advanced stats: %w", err)
+		}
+	}
+	return nil
+}
+
+// uncommentDoc finds and replaces commented out HTML sections.
+func uncommentDoc(doc *goquery.Document) *goquery.Document {
+	doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) {
+		s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool {
+			return goquery.NodeName(s) == "#comment"
+		}).Each(func(i int, s *goquery.Selection) {
+			uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text()))
+			// FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string.
+			s.ReplaceWithSelection(uncommented.Find("body").Children())
+		})
+	})
+	return doc
+}
+
+// getModelColumns is a placeholder for a more robust reflection-based column name generator.
+// For now, it returns hardcoded lists.
+func getModelColumns(model interface{}) []string {
+	switch model.(type) {
+	case *models.PlayerGameBasicStat:
+		return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"}
+	case *models.PlayerGameAdvStat:
+		return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"}
+	case *models.TeamGameBasicStat:
+		return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"}
+	case *models.TeamGameAdvStat:
+		return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"}
+	}
+	return []string{}
+}

services/game_scrape_service.go

@@ -0,0 +1,190 @@
+// File: services/game_scrape_service.go
+package services
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const gameScheduleURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_games-%s.html"
+
+// FetchAndStoreGameSchedule scrapes the game schedule for a given season and month.
+// The month should be the full lowercase name, e.g., "october", "november".
+// If db is nil, it will perform a "dry run" and print the parsed data to the console.
+func FetchAndStoreGameSchedule(db *gorm.DB, season int, month string) error {
+	url := fmt.Sprintf(gameScheduleURLFmt, season, month)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to fetch schedule for %s %d: %w", month, season, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("⚠️  Skipping schedule for %s %d (Status: %s)", month, season, resp.Status)
+		return nil // Not a fatal error, just no data for this month.
+	}
+
+	htmlBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read response body for %s %d: %w", month, season, err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
+	if err != nil {
+		return fmt.Errorf("failed to parse HTML for %s %d: %w", month, season, err)
+	}
+
+	table := doc.Find("table#schedule")
+	if table.Length() == 0 {
+		// Sometimes the content is commented out
+		commentNode := doc.Find("#all_schedule").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+			return goquery.NodeName(s) == "#comment"
+		})
+		if commentNode.Length() > 0 {
+			commentedHTML := commentNode.Nodes[0].FirstChild.Data
+			innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
+			if err != nil {
+				return fmt.Errorf("failed to parse commented schedule HTML: %w", err)
+			}
+			table = innerDoc.Find("table#schedule")
+		}
+	}
+
+	if table.Length() == 0 {
+		log.Printf("No schedule table found for %s %d.", month, season)
+		return nil
+	}
+
+	var gamesToUpsert []models.Game
+	table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
+		// Skip table header rows that are sometimes repeated in the body
+		if row.Find("th.poptip").Length() > 1 {
+			return
+		}
+
+		// Skip rows that don't represent games (e.g., placeholder rows)
+		if row.Find(`[data-stat="visitor_team_name"]`).Text() == "" {
+			return
+		}
+
+		var game models.Game
+		var gameID string
+
+		// Extract GameID from the box score link, which is the most reliable unique key
+		boxScoreCell := row.Find(`td[data-stat="box_score_text"] a`)
+		if href, exists := boxScoreCell.Attr("href"); exists {
+			parts := strings.Split(href, "/")
+			fileName := parts[len(parts)-1]
+			gameID = strings.TrimSuffix(fileName, ".html")
+		}
+
+		// If there's no box score link, it's likely a future game, we can skip it or handle differently
+		if gameID == "" {
+			return
+		}
+		game.GameID = gameID
+
+		// Get the date part from the 'csk' attribute for accuracy
+		dateCsk, _ := row.Find(`th[data-stat="date_game"]`).Attr("csk")
+		var datePart string
+		if len(dateCsk) >= 8 {
+			datePart = dateCsk[:8]
+		} else {
+			log.Printf("Could not parse date from invalid 'csk' attribute: %s. Skipping.", dateCsk)
+			return
+		}
+
+		// Get the start time string
+		startTimeET := row.Find(`td[data-stat="game_start_time"]`).Text()
+		if startTimeET == "" {
+			// Skip games without a start time, as they can't be parsed accurately
+			log.Printf("Could not find start time for game %s. Skipping.", gameID)
+			return
+		}
+
+		// Load the US/Eastern timezone to correctly handle ET/EST/EDT
+		eastern, err := time.LoadLocation("America/New_York")
+		if err != nil {
+			log.Printf("FATAL: Could not load America/New_York timezone: %v", err)
+			// This is a system-level error, so we stop the row processing here.
+			// The function will continue and process any games already parsed.
+			return
+		}
+
+		// Combine date and time and parse together in the correct timezone.
+		// The layout "3:04p" handles times like "7:30p".
+		fullDateTimeString := datePart + startTimeET
+		layout := "200601023:04p"
+		gameDate, err := time.ParseInLocation(layout, fullDateTimeString, eastern)
+		if err != nil {
+			log.Printf("Could not parse combined date-time for game %s (value: '%s'): %v. Skipping.", gameID, fullDateTimeString, err)
+			return
+		}
+		game.Date = gameDate
+		game.StartTimeET = startTimeET // Keep the original string as well
+
+		game.VisitorTeam = row.Find(`td[data-stat="visitor_team_name"] a`).Text()
+		game.VisitorPTS = mustAtoi(row.Find(`td[data-stat="visitor_pts"]`).Text())
+		game.HomeTeam = row.Find(`td[data-stat="home_team_name"] a`).Text()
+		game.HomePTS = mustAtoi(row.Find(`td[data-stat="home_pts"]`).Text())
+		game.BoxScoreURL, _ = boxScoreCell.Attr("href")
+		game.GameDuration = row.Find(`td[data-stat="game_duration"]`).Text()
+		game.Arena = row.Find(`td[data-stat="arena_name"]`).Text()
+		game.IsPlayoff = strings.Contains(row.Find(`td[data-stat="game_remarks"]`).Text(), "Playoffs")
+
+		gamesToUpsert = append(gamesToUpsert, game)
+	})
+
+	if len(gamesToUpsert) > 0 {
+		// If the db connection is nil, we're in test/debug mode. Print to console.
+		if db == nil {
+			log.Println("--- RUNNING IN DRY-RUN MODE ---")
+			for _, game := range gamesToUpsert {
+				// Use %+v to print the struct with field names for clarity
+				log.Printf("Game Data: %+v\n", game)
+			}
+			log.Printf("--- WOULD INSERT %d RECORDS ---", len(gamesToUpsert))
+			return nil // End execution for dry-run
+		}
+
+		log.Printf("Attempting to batch upsert %d games for %s %d...", len(gamesToUpsert), month, season)
+
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}},
+			DoUpdates: clause.AssignmentColumns(allGameColumns()),
+		}).Create(&gamesToUpsert).Error; err != nil {
+			log.Printf("Failed to batch upsert games: %v", err)
+			return err
+		}
+		log.Printf("✅ Successfully batch upserted %d game records for %s %d.", len(gamesToUpsert), month, season)
+	} else {
+		log.Printf("No game data found to import for %s %d.", month, season)
+	}
+
+	return nil
+}
+
+// allGameColumns returns a list of all column names in the Game model for the upsert operation.
+// This ensures that if a record exists, all its fields are updated with the new data.
+func allGameColumns() []string {
+	return []string{
+		"date", "is_playoff", "start_time_et", "arena", "visitor_team",
+		"visitor_pts", "home_team", "home_pts", "game_duration", "box_score_url",
+		"updated_at",
+	}
+}

services/helpers.go

@@ -8,6 +8,16 @@ func mustAtoi(s string) int {
     return i
 }
 
+// mustAtoiWithSign handles strings that might have a "+" or "-" sign.
+func mustAtoiWithSign(s string) int {
+	if s == "" {
+		return 0
+	}
+	// The strconv.Atoi function handles the sign automatically.
+	i, _ := strconv.Atoi(s)
+	return i
+}
+
 // mustParseFloat parses s into a float64, or returns 0.0 on error.
 func mustParseFloat(s string) float64 {
     f, _ := strconv.ParseFloat(s, 64)