Merge pull request #29 from nprasad2077/games

Games

Dockerfile

@@ -4,7 +4,8 @@ FROM golang:1.24-bullseye AS builder
 ENV CGO_ENABLED=1
 ENV GOOS=linux
 # Changed from Apple Silicon arm64 config.
-ENV GOARCH=amd64
+# ENV GOARCH=amd64
+ENV GOARCH=arm64
 
 WORKDIR /app
 

README.md

@@ -29,3 +29,10 @@ swag init -g main.go -o docs
 ```bash
 go run loadtest.go -n 100 -c 10 -url "http://127.0.0.1:8080/api/playeradvancedstats?page=1&pageSize=20" -log results.log -key "xxx"
 ```
+
+
+### Local Environment
+
+```bash
+docker-compose -f docker-compose.local.yml up --build -d
+```

config/database.go

@@ -1,3 +1,5 @@
+// File: config/database.go
+
 package config
 
 import (
@@ -12,7 +14,7 @@ import (
 )
 
 func InitDB(shouldMigrate bool) *gorm.DB {
-	// CHANGE: Build the DSN from environment variables
+	// ... (DSN setup code is unchanged) ...
 	dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC",
 		os.Getenv("DB_HOST"),
 		os.Getenv("DB_USER"),
@@ -21,13 +23,32 @@ func InitDB(shouldMigrate bool) *gorm.DB {
 		os.Getenv("DB_PORT"),
 	)
 
-	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) // <- CHANGE: Use the postgres driver
+	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
 	if err != nil {
 		log.Fatalf("failed to connect database: %v", err)
 	}
 	metrics.DBOperationsTotal.WithLabelValues("connect", "database").Inc()
 
 	if shouldMigrate {
+		// --- ADD THIS BLOCK TO DROP STALE TABLES ---
+		// Drop tables in reverse order of dependency (children first).
+		// This ensures a clean migration every time the import process runs.
+		// log.Println("⚠️ Dropping existing game-related tables for a clean migration...")
+		// if err := db.Migrator().DropTable(
+		// 	&models.LineScore{},
+		// 	&models.PlayerGameBasicStat{},
+		// 	&models.PlayerGameAdvStat{},
+		// 	&models.TeamGameBasicStat{},
+		// 	&models.TeamGameAdvStat{},
+		// 	&models.Game{}, // Drop parent table last
+		// ); err != nil {
+		// 	log.Fatalf("failed to drop tables: %v", err)
+		// }
+		// log.Println("✅ Tables dropped successfully.")
+		// --- END OF ADDED BLOCK ---
+
+
+		// Your existing AutoMigrate calls will now work correctly
 		if err := db.AutoMigrate(&models.PlayerAdvancedStat{}); err != nil {
 			log.Fatalf("migrate PlayerAdvancedStat: %v", err)
 		}
@@ -40,6 +61,17 @@ func InitDB(shouldMigrate bool) *gorm.DB {
 		if err := db.AutoMigrate(&models.APIKey{}); err != nil {
 			log.Fatalf("migrate APIKey: %v", err)
 		}
+		// Game Models will be recreated with the correct schema
+		if err := db.AutoMigrate(
+			&models.Game{},
+			&models.LineScore{},
+			&models.PlayerGameBasicStat{},
+			&models.PlayerGameAdvStat{},
+			&models.TeamGameBasicStat{},
+			&models.TeamGameAdvStat{},
+		); err != nil {
+			log.Fatalf("migrate game models: %v", err)
+		}
 		metrics.DBOperationsTotal.WithLabelValues("migrate", "database").Inc()
 	}
 

docker-compose.local.yml

@@ -1,49 +1,72 @@
-# docker-compose.local.yml
+version: '3.8'
-#
-# Use this file to run local services (API, Nginx, etc.) while
-# connecting to a REMOTE database.
-#
-# This version includes the 'db-init' service. Note that this service
-# will attempt to run its 'import-data' command every time you execute
-# 'docker-compose up'.
 
 services:
-  # This service runs the 'import-data' command against the remote database
+  # NEW: Add a dedicated PostgreSQL database service
-  # specified in your '.env.local' file. It has no 'depends_on' since there
+  postgres:
-  # is no local postgres container to wait for.
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_NAME}
+    ports:
+      - '${DB_PORT}:5432' # Expose DB port to host machine for local tools
+    volumes:
+      - postgres_data:/var/lib/postgresql/data # Persist data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -d ${DB_NAME}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - api.network
+
+  # MODIFIED: This service now runs migrations and imports against the live Postgres DB
+  # "import-data" on initial run to migrate and import data.
   db-init:
     build: .
-    env_file: .env.local
+    env_file: .env
     command: ["/nba_go", "import-data"]
+    depends_on:
+      postgres:
+        condition: service_healthy
     networks:
       - api.network
     restart: "no"
 
-  # API services now have no 'depends_on' section.
+  # MODIFIED: All API services now depend on postgres being healthy.
-  # They will start and immediately try to connect to the DB_HOST in your .env file.
+  # The volume mount for './data' is no longer needed.
   api1:
     build: .
-    env_file: .env.local
+    env_file: .env
     ports:
       - '5001:5000'
+    depends_on:
+      postgres:
+        condition: service_healthy
     networks:
       - api.network
     restart: always
 
   api2:
     build: .
-    env_file: .env.local
+    env_file: .env
     ports:
       - '5002:5000'
+    depends_on:
+      postgres:
+        condition: service_healthy
     networks:
       - api.network
     restart: always
 
   api3:
     build: .
-    env_file: .env.local
+    env_file: .env
     ports:
       - '5003:5000'
+    depends_on:
+      postgres:
+        condition: service_healthy
     networks:
       - api.network
     restart: always
@@ -57,7 +80,7 @@ services:
       - api2
       - api3
     ports:
-      - '8080:8080'
+      - '8081:8080'
     networks:
       - api.network
 
@@ -76,6 +99,8 @@ services:
     networks:
       - api.network
     restart: unless-stopped
+    logging:
+      driver: "none"
 
   grafana:
     image: grafana/grafana:latest
@@ -91,9 +116,11 @@ services:
     networks:
       - api.network
     restart: unless-stopped
+    logging:
+      driver: none
 
 volumes:
-  # The 'postgres_data' volume is not needed in this configuration.
+  postgres_data: # New volume for Postgres
   prometheus_data:
   grafana_data:
 

docs/docs.go

@@ -477,7 +477,7 @@ var SwaggerInfo = &swag.Spec{
 	Version:          "1.0",
 	Host:             "",
 	BasePath:         "/",
-	Schemes:          []string{"https"},
+	Schemes:          []string{"http"},
 	Title:            "NBA_Go API",
 	Description:      "Stats service, now with public access!",
 	InfoInstanceName: "swagger",

docs/swagger.json

@@ -1,6 +1,6 @@
 {
     "schemes": [
-        "https"
+        "http"
     ],
     "swagger": "2.0",
     "info": {

docs/swagger.yaml

@@ -303,7 +303,7 @@ paths:
       - PlayerTotals
       x-order: 1
 schemes:
-- https
+- http
 swagger: "2.0"
 tags:
 - name: PlayerTotals

import.go

@@ -4,14 +4,14 @@ import (
 	"log"
 	"time"
 
-	"gorm.io/gorm"
 	"github.com/nprasad2077/NBA_Go/services"
 	"github.com/nprasad2077/NBA_Go/utils"
+	"gorm.io/gorm"
 )
 
 // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
 func importPlayerAdvanced(db *gorm.DB) {
-	for season := 1991; season <= 2002; season++ {
+	for season := 2024; season <= 2025; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
@@ -23,42 +23,85 @@ func importPlayerAdvanced(db *gorm.DB) {
 
 // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
 func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 1991; season <= 2002; season++ {
+	for season := 2024; season <= 2025; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
 		log.Printf("Advanced Playoffs import for season: %d", season)
 		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1250 * time.Millisecond)
+		utils.SleepWithJitter(1500 * time.Millisecond)
 	}
 }
 
 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-    for season := 1991; season <= 2002; season++ {
+	for season := 2024; season <= 2025; season++ {
-        if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
+		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
-            log.Printf("scraped totals import failed for %d: %v", season, err)
+			log.Printf("scraped totals import failed for %d: %v", season, err)
-        }
+		}
 		log.Printf("Player Totals import for season: %d", season)
-        time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1500 * time.Millisecond)
+		utils.SleepWithJitter(1250 * time.Millisecond)
-    }
+	}
 }
 
 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-    for season := 1991; season <= 2002; season++ {
+	for season := 2024; season <= 2025; season++ {
-        if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
+		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
-            log.Printf("scraped playoffs import failed for %d: %v", season, err)
+			log.Printf("scraped playoffs import failed for %d: %v", season, err)
-        }
+		}
 		log.Printf("Player Playoffs Totals import for season: %d", season)
-        time.Sleep(1100 * time.Millisecond)
+		time.Sleep(1100 * time.Millisecond)
-		utils.SleepWithJitter(1750 * time.Millisecond)
+		utils.SleepWithJitter(1700 * time.Millisecond)
-    }
+	}
 }
 
+// importGameSchedules fetches and stores game schedules
+func importGameSchedules(db *gorm.DB) {
+	// An NBA season typically runs from October to June
+	months := []string{
+		"october", "november", "december", "january",
+		"february", "march", "april", "may", "june",
 
+		// "february", "march", "april", "may", "june",
+	}
 
+	for season := 2018; season <= 2025; season++ {
+		log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
+		for _, month := range months {
+			// The service will print a warning and skip if a month has no data (e.g. May/June for a season not yet finished)
+			if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil {
+				// Log the error but continue to the next month/season
+				log.Printf("Game schedule import failed for %s %d: %v", month, season, err)
+			}
+			log.Printf("Game schedule import for %s, %d complete.", month, season)
+			// Respectful delay between requests
+			time.Sleep(1100 * time.Millisecond)
+			utils.SleepWithJitter(1800 * time.Millisecond)
+		}
+		log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
+	}
+}
+
+// importBoxScores fetches and stores all box score data (line scores, player/team stats)
+// for games within a recent date range.
+func importBoxScores(db *gorm.DB) {
+	// Define the date range for the import.
+
+	// The format is: time.Date(year, month, day, hour, min, sec, nsec, location)
+    // to := time.Date(2019, time.June, 14, 0, 0, 0, 0, time.UTC)
+ 	to := time.Now()
+	from := to.AddDate(-1, 0, 0) // 0 years, -3 months, 0 days
+
+	log.Printf("--- Starting Box Score Data Import from %s to %s ---", from.Format("2006-01-02"), to.Format("2006-01-02"))
+
+	if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
+		log.Fatalf("Box score import failed: %v", err)
+	}
+
+	log.Printf("--- Finished Box Score Data Import ---")
+}
 
 // importPlayerShotChart fetches shot-charts for every known player
 // func importPlayerShotChart(db *gorm.DB) {

main.go

@@ -1,7 +1,8 @@
 // @title       NBA_Go API
 // @version     1.0
 // @description Stats service, now with public access!
-// @schemes     https
+// //@schemes     https
+// @schemes     http
 // @BasePath    /
 //
 // @tag.name    PlayerTotals
@@ -45,7 +46,7 @@ import (
 func main() {
 	// ——— One-off import-data mode ———
 	if len(os.Args) > 1 && os.Args[1] == "import-data" {
-		// Run all migrations + import steps exactly once
+		// Run all DB migrations + import steps exactly once
 		db := config.InitDB(true)
 
 		importPlayerAdvanced(db)
@@ -60,6 +61,13 @@ func main() {
 		importPlayerTotalsPlayoffsScrape(db)
 		log.Println("🎉 Player Playoffs (scraped) Import completed successfully")
 
+		importGameSchedules(db)
+		log.Println("🎉 Game Imports completed successfully 🏀")
+
+		importBoxScores(db)
+		log.Println("🎉 Related Box Score Imports completed successfully 📦")
+
+
 		log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
 		return
 	}

models/game.go

@@ -0,0 +1,172 @@
+package models
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+type Game struct {
+	ID           uint      `gorm:"primaryKey" swaggerignore:"true"`
+	GameID       string    `gorm:"not null;uniqueIndex" json:"gameId"` // The field you want to use
+	Date         time.Time `gorm:"not null;index" json:"date"`
+	IsPlayoff    bool      `gorm:"not null;default:false;index" json:"isPlayoff"`
+	StartTimeET  string    `json:"startTimeET"`
+	Arena        string    `json:"arena"`
+	VisitorTeam  string    `gorm:"not null" json:"visitorTeam"`
+	VisitorPTS   int       `json:"visitorPts"`
+	HomeTeam     string    `gorm:"not null" json:"homeTeam"`
+	HomePTS      int       `json:"homePts"`
+	GameDuration string    `json:"gameDuration"`
+	BoxScoreURL  string    `json:"boxScoreUrl"`
+
+	// Associations (Updated with explicit foreign key tags)
+	LineScores           []LineScore           `gorm:"foreignKey:GameID;references:GameID" json:"lineScores"`
+	PlayerGameBasicStats []PlayerGameBasicStat `gorm:"foreignKey:GameID;references:GameID" json:"playerGameBasicStats"`
+	PlayerGameAdvStats   []PlayerGameAdvStat   `gorm:"foreignKey:GameID;references:GameID" json:"playerGameAdvStats"`
+	TeamGameBasicStats   []TeamGameBasicStat   `gorm:"foreignKey:GameID;references:GameID" json:"teamGameBasicStats"`
+	TeamGameAdvStats     []TeamGameAdvStat     `gorm:"foreignKey:GameID;references:GameID" json:"teamGameAdvStats"`
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}
+
+// LineScore holds the scoring for each team by quarter for a specific game.
+type LineScore struct {
+	ID     uint   `gorm:"primaryKey" swaggerignore:"true"`
+	GameID string `gorm:"not null;uniqueIndex:idx_line_score,priority:1" json:"gameId"`
+	Team   string `gorm:"not null;uniqueIndex:idx_line_score,priority:2" json:"team"`
+	Q1     int    `json:"q1"`
+	Q2     int    `json:"q2"`
+	Q3     int    `json:"q3"`
+	Q4     int    `json:"q4"`
+	// Overtime scores; will be 0 if the game did not go to the respective OT period.
+	OT1   int `json:"ot1"`
+	OT2   int `json:"ot2"`
+	OT3   int `json:"ot3"`
+	Total int `json:"total"`
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}
+
+// PlayerGameBasicStat contains the basic box score statistics for a single player in a single game.
+type PlayerGameBasicStat struct {
+	ID            uint    `gorm:"primaryKey" swaggerignore:"true"`
+	GameID        string  `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"gameId"`
+	PlayerID      string  `gorm:"not null;uniqueIndex:idx_player_game_basic" json:"playerId"`
+	PlayerName    string  `json:"playerName"`
+	Team          string  `gorm:"not null" json:"team"`
+	Status        string  `json:"status"` // e.g., "Starter", "Bench", "Did Not Play"
+	MP            string  `json:"mp"`     // Minutes Played, e.g., "38:07"
+	FG            int     `json:"fg"`
+	FGA           int     `json:"fga"`
+	FGPercent     float64 `json:"fgPercent"`
+	ThreeP        int     `json:"threeP"`
+	ThreePA       int     `json:"threePa"`
+	ThreePPercent float64 `json:"threePPercent"`
+	FT            int     `json:"ft"`
+	FTA           int     `json:"fta"`
+	FTPercent     float64 `json:"ftPercent"`
+	ORB           int     `json:"orb"`
+	DRB           int     `json:"drb"`
+	TRB           int     `json:"trb"`
+	AST           int     `json:"ast"`
+	STL           int     `json:"stl"`
+	BLK           int     `json:"blk"`
+	TOV           int     `json:"tov"`
+	PF            int     `json:"pf"`
+	PTS           int     `json:"pts"`
+	GmSc          float64 `json:"gmSc"` // Game Score
+	PlusMinus     int     `json:"plusMinus"`
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}
+
+// PlayerGameAdvStat contains the advanced box score statistics for a single player in a single game.
+type PlayerGameAdvStat struct {
+	ID         uint    `gorm:"primaryKey" swaggerignore:"true"`
+	GameID     string  `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"gameId"`
+	PlayerID   string  `gorm:"not null;uniqueIndex:idx_player_game_adv" json:"playerId"`
+	PlayerName string  `json:"playerName"`
+	Team       string  `gorm:"not null" json:"team"`
+	MP         string  `json:"mp"` // Minutes Played
+	TSPercent  float64 `json:"tsPercent"`
+	EFGPercent float64 `json:"efgPercent"`
+	ThreePAr   float64 `json:"threePAr"`
+	FTr        float64 `json:"fTr"`
+	ORBPercent float64 `json:"orbPercent"`
+	DRBPercent float64 `json:"drbPercent"`
+	TRBPercent float64 `json:"trbPercent"`
+	ASTPercent float64 `json:"astPercent"`
+	STLPercent float64 `json:"stlPercent"`
+	BLKPercent float64 `json:"blkPercent"`
+	TOVPercent float64 `json:"tovPercent"`
+	USGPercent float64 `json:"usgPercent"`
+	ORtg       int     `json:"oRtg"`
+	DRtg       int     `json:"dRtg"`
+	BPM        float64 `json:"bpm"` // Box Plus/Minus
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}
+
+// TeamGameBasicStat holds the total basic stats for a team in a single game.
+type TeamGameBasicStat struct {
+	ID            uint    `gorm:"primaryKey" swaggerignore:"true"`
+	GameID        string  `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"gameId"`
+	Team          string  `gorm:"not null;uniqueIndex:idx_team_game_basic" json:"team"`
+	MP            int     `json:"mp"` // Total minutes, usually 240 for a regulation game
+	FG            int     `json:"fg"`
+	FGA           int     `json:"fga"`
+	FGPercent     float64 `json:"fgPercent"`
+	ThreeP        int     `json:"threeP"`
+	ThreePA       int     `json:"threePa"`
+	ThreePPercent float64 `json:"threePPercent"`
+	FT            int     `json:"ft"`
+	FTA           int     `json:"fta"`
+	FTPercent     float64 `json:"ftPercent"`
+	ORB           int     `json:"orb"`
+	DRB           int     `json:"drb"`
+	TRB           int     `json:"trb"`
+	AST           int     `json:"ast"`
+	STL           int     `json:"stl"`
+	BLK           int     `json:"blk"`
+	TOV           int     `json:"tov"`
+	PF            int     `json:"pf"`
+	PTS           int     `json:"pts"`
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}
+
+// TeamGameAdvStat holds the total advanced stats for a team in a single game.
+type TeamGameAdvStat struct {
+	ID         uint    `gorm:"primaryKey" swaggerignore:"true"`
+	GameID     string  `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"gameId"`
+	Team       string  `gorm:"not null;uniqueIndex:idx_team_game_adv" json:"team"`
+	MP         int     `json:"mp"`
+	TSPercent  float64 `json:"tsPercent"`
+	EFGPercent float64 `json:"efgPercent"`
+	ThreePAr   float64 `json:"threePAr"`
+	FTr        float64 `json:"fTr"`
+	ORBPercent float64 `json:"orbPercent"`
+	DRBPercent float64 `json:"drbPercent"`
+	TRBPercent float64 `json:"trbPercent"`
+	ASTPercent float64 `json:"astPercent"`
+	STLPercent float64 `json:"stlPercent"`
+	BLKPercent float64 `json:"blkPercent"`
+	TOVPercent float64 `json:"tovPercent"`
+	USGPercent float64 `json:"usgPercent"` // Will be ~100.0 for a team
+	ORtg       float64 `json:"oRtg"`
+	DRtg       float64 `json:"dRtg"`
+
+	CreatedAt time.Time      `swaggerignore:"true"`
+	UpdatedAt time.Time      `swaggerignore:"true"`
+	DeletedAt gorm.DeletedAt `gorm:"index" swaggerignore:"true"`
+}

services/box_score_scrape_service.go

@@ -0,0 +1,312 @@
+package services
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"github.com/nprasad2077/NBA_Go/utils"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const boxScoreURLBase = "https://www.basketball-reference.com"
+
+// uncommentDoc finds and replaces commented out HTML sections.
+func uncommentDoc(doc *goquery.Document) *goquery.Document {
+	doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+		return goquery.NodeName(s) == "#comment"
+	}).Each(func(i int, s *goquery.Selection) {
+		// Use .Data on the underlying html.Node to get the comment content.
+		commentText := s.Nodes[0].Data
+		if strings.Contains(commentText, "<table") {
+			// Replace the comment node with its content.
+			s.ReplaceWithHtml(commentText)
+		}
+	})
+	return doc
+}
+
+// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
+func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
+	var games []models.Game
+	// Query the database for games within the specified date range.
+	if err := db.Where("date >= ? AND date < ?", from, to).Find(&games).Error; err != nil {
+		return fmt.Errorf("failed to query games from DB: %w", err)
+	}
+
+	log.Printf("Found %d games to process in the specified date range.", len(games))
+
+	for _, game := range games {
+		log.Printf("Processing game: %s", game.GameID)
+		fullURL := boxScoreURLBase + game.BoxScoreURL
+		if err := scrapeBoxScorePage(db, fullURL, game.GameID); err != nil {
+			// Log the error but continue to the next game
+			log.Printf("Error processing box score for game %s: %v", game.GameID, err)
+		}
+		// Be a good internet citizen and pause between requests.
+		utils.SleepWithJitter(2300 * time.Millisecond)
+	}
+	return nil
+}
+
+// scrapeBoxScorePage handles fetching and parsing a single box score page.
+func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("received non-200 status code: %s", resp.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// 1. Uncomment all tables in the document first.
+	doc = uncommentDoc(doc)
+
+	// 2. Call the dedicated service to handle line scores.
+	if err := FetchAndStoreLineScore(db, doc, gameID); err != nil {
+		log.Printf("Error processing line score for game %s: %v", gameID, err)
+	}
+
+	// 3. The existing box score parser will now work because its tables are visible.
+	if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
+func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
+	var allPlayerBasicStats []models.PlayerGameBasicStat
+	var allPlayerAdvStats []models.PlayerGameAdvStat
+	var allTeamBasicStats []models.TeamGameBasicStat
+	var allTeamAdvStats []models.TeamGameAdvStat
+
+	// Use a CSS attribute selector to find all box score tables for both teams.
+	doc.Find(`table[id^="box-"][id$="-game-basic"], table[id^="box-"][id$="-game-advanced"]`).Each(func(i int, table *goquery.Selection) {
+		tableID, _ := table.Attr("id")
+		isAdvanced := strings.Contains(tableID, "-advanced")
+		teamAbbr := strings.TrimSuffix(strings.TrimPrefix(tableID, "box-"), "-game-basic")
+		teamAbbr = strings.TrimSuffix(teamAbbr, "-game-advanced")
+
+		// Process player rows
+		table.Find("tbody tr").Each(func(j int, row *goquery.Selection) {
+			playerID, exists := row.Find("th").Attr("data-append-csv")
+			if !exists || playerID == "" {
+				return // Not a player row
+			}
+
+			// Handle "Did Not Play" or other statuses
+			reason := row.Find(`td[data-stat="reason"]`)
+			status := "Played"
+			if reason.Length() > 0 {
+				status = reason.Text()
+			}
+
+			if !isAdvanced {
+				stat := parsePlayerBasicStat(row, gameID, playerID, teamAbbr, status)
+				allPlayerBasicStats = append(allPlayerBasicStats, stat)
+			} else {
+				stat := parsePlayerAdvStat(row, gameID, playerID, teamAbbr, status)
+				allPlayerAdvStats = append(allPlayerAdvStats, stat)
+			}
+		})
+
+		// Process team total row
+		table.Find("tfoot tr").Each(func(j int, row *goquery.Selection) {
+			if !isAdvanced {
+				stat := parseTeamBasicStat(row, gameID, teamAbbr)
+				allTeamBasicStats = append(allTeamBasicStats, stat)
+			} else {
+				stat := parseTeamAdvStat(row, gameID, teamAbbr)
+				allTeamAdvStats = append(allTeamAdvStats, stat)
+			}
+		})
+	})
+
+	// Batch upsert all collected stats
+	if err := batchUpsertAll(db, allPlayerBasicStats, allPlayerAdvStats, allTeamBasicStats, allTeamAdvStats); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// --- Parsing Helper Functions ---
+
+func parsePlayerBasicStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameBasicStat {
+	return models.PlayerGameBasicStat{
+		GameID:        gameID,
+		PlayerID:      playerID,
+		PlayerName:    row.Find(`th[data-stat="player"] a`).Text(),
+		Team:          team,
+		Status:        status,
+		MP:            row.Find(`td[data-stat="mp"]`).Text(),
+		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
+		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
+		FGPercent:     mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
+		ThreeP:        mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
+		ThreePA:       mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
+		ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
+		FT:            mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
+		FTA:           mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
+		FTPercent:     mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
+		ORB:           mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
+		DRB:           mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
+		TRB:           mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
+		AST:           mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
+		STL:           mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
+		BLK:           mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
+		TOV:           mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
+		PF:            mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
+		PTS:           mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
+		GmSc:          mustParseFloat(row.Find(`td[data-stat="game_score"]`).Text()),
+		PlusMinus:     mustAtoiWithSign(row.Find(`td[data-stat="plus_minus"]`).Text()),
+	}
+}
+
+func parsePlayerAdvStat(row *goquery.Selection, gameID, playerID, team, status string) models.PlayerGameAdvStat {
+	return models.PlayerGameAdvStat{
+		GameID:     gameID,
+		PlayerID:   playerID,
+		PlayerName: row.Find(`th[data-stat="player"] a`).Text(),
+		Team:       team,
+		MP:         row.Find(`td[data-stat="mp"]`).Text(),
+		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
+		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
+		ThreePAr:   mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
+		FTr:        mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
+		ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
+		DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
+		TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
+		ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
+		STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
+		BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
+		TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
+		USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
+		ORtg:       mustAtoi(row.Find(`td[data-stat="off_rtg"]`).Text()),
+		DRtg:       mustAtoi(row.Find(`td[data-stat="def_rtg"]`).Text()),
+		BPM:        mustParseFloat(row.Find(`td[data-stat="bpm"]`).Text()),
+	}
+}
+
+func parseTeamBasicStat(row *goquery.Selection, gameID, team string) models.TeamGameBasicStat {
+	return models.TeamGameBasicStat{
+		GameID:        gameID,
+		Team:          team,
+		MP:            mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
+		FG:            mustAtoi(row.Find(`td[data-stat="fg"]`).Text()),
+		FGA:           mustAtoi(row.Find(`td[data-stat="fga"]`).Text()),
+		FGPercent:     mustParseFloat(row.Find(`td[data-stat="fg_pct"]`).Text()),
+		ThreeP:        mustAtoi(row.Find(`td[data-stat="fg3"]`).Text()),
+		ThreePA:       mustAtoi(row.Find(`td[data-stat="fg3a"]`).Text()),
+		ThreePPercent: mustParseFloat(row.Find(`td[data-stat="fg3_pct"]`).Text()),
+		FT:            mustAtoi(row.Find(`td[data-stat="ft"]`).Text()),
+		FTA:           mustAtoi(row.Find(`td[data-stat="fta"]`).Text()),
+		FTPercent:     mustParseFloat(row.Find(`td[data-stat="ft_pct"]`).Text()),
+		ORB:           mustAtoi(row.Find(`td[data-stat="orb"]`).Text()),
+		DRB:           mustAtoi(row.Find(`td[data-stat="drb"]`).Text()),
+		TRB:           mustAtoi(row.Find(`td[data-stat="trb"]`).Text()),
+		AST:           mustAtoi(row.Find(`td[data-stat="ast"]`).Text()),
+		STL:           mustAtoi(row.Find(`td[data-stat="stl"]`).Text()),
+		BLK:           mustAtoi(row.Find(`td[data-stat="blk"]`).Text()),
+		TOV:           mustAtoi(row.Find(`td[data-stat="tov"]`).Text()),
+		PF:            mustAtoi(row.Find(`td[data-stat="pf"]`).Text()),
+		PTS:           mustAtoi(row.Find(`td[data-stat="pts"]`).Text()),
+	}
+}
+
+func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGameAdvStat {
+	return models.TeamGameAdvStat{
+		GameID:     gameID,
+		Team:       team,
+		MP:         mustAtoi(row.Find(`td[data-stat="mp"]`).Text()),
+		TSPercent:  mustParseFloat(row.Find(`td[data-stat="ts_pct"]`).Text()),
+		EFGPercent: mustParseFloat(row.Find(`td[data-stat="efg_pct"]`).Text()),
+		ThreePAr:   mustParseFloat(row.Find(`td[data-stat="fg3a_per_fga_pct"]`).Text()),
+		FTr:        mustParseFloat(row.Find(`td[data-stat="fta_per_fga_pct"]`).Text()),
+		ORBPercent: mustParseFloat(row.Find(`td[data-stat="orb_pct"]`).Text()),
+		DRBPercent: mustParseFloat(row.Find(`td[data-stat="drb_pct"]`).Text()),
+		TRBPercent: mustParseFloat(row.Find(`td[data-stat="trb_pct"]`).Text()),
+		ASTPercent: mustParseFloat(row.Find(`td[data-stat="ast_pct"]`).Text()),
+		STLPercent: mustParseFloat(row.Find(`td[data-stat="stl_pct"]`).Text()),
+		BLKPercent: mustParseFloat(row.Find(`td[data-stat="blk_pct"]`).Text()),
+		TOVPercent: mustParseFloat(row.Find(`td[data-stat="tov_pct"]`).Text()),
+		USGPercent: mustParseFloat(row.Find(`td[data-stat="usg_pct"]`).Text()),
+		ORtg:       mustParseFloat(row.Find(`td[data-stat="off_rtg"]`).Text()),
+		DRtg:       mustParseFloat(row.Find(`td[data-stat="def_rtg"]`).Text()),
+	}
+}
+
+// --- DB and Utility Functions ---
+
+func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
+	if len(pbs) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
+		}).Create(&pbs).Error; err != nil {
+			return fmt.Errorf("failed to upsert player basic stats: %w", err)
+		}
+	}
+	if len(pas) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
+		}).Create(&pas).Error; err != nil {
+			return fmt.Errorf("failed to upsert player advanced stats: %w", err)
+		}
+	}
+	if len(tbs) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
+		}).Create(&tbs).Error; err != nil {
+			return fmt.Errorf("failed to upsert team basic stats: %w", err)
+		}
+	}
+	if len(tas) > 0 {
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+			DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
+		}).Create(&tas).Error; err != nil {
+			return fmt.Errorf("failed to upsert team advanced stats: %w", err)
+		}
+	}
+	return nil
+}
+
+
+
+// getModelColumns is a placeholder for a more robust reflection-based column name generator.
+// For now, it returns hardcoded lists.
+func getModelColumns(model interface{}) []string {
+	switch model.(type) {
+	case *models.PlayerGameBasicStat:
+		return []string{"player_name", "team", "status", "mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts", "gm_sc", "plus_minus"}
+	case *models.PlayerGameAdvStat:
+		return []string{"player_name", "team", "mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg", "bpm"}
+	case *models.TeamGameBasicStat:
+		return []string{"mp", "fg", "fga", "fg_percent", "three_p", "three_pa", "three_p_percent", "ft", "fta", "ft_percent", "orb", "drb", "trb", "ast", "stl", "blk", "tov", "pf", "pts"}
+	case *models.TeamGameAdvStat:
+		return []string{"mp", "ts_percent", "efg_percent", "three_p_ar", "f_tr", "orb_percent", "drb_percent", "trb_percent", "ast_percent", "stl_percent", "blk_percent", "tov_percent", "usg_percent", "o_rtg", "d_rtg"}
+	}
+	return []string{}
+}

services/game_scrape_service.go

@@ -0,0 +1,190 @@
+// File: services/game_scrape_service.go
+package services
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const gameScheduleURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_games-%s.html"
+
+// FetchAndStoreGameSchedule scrapes the game schedule for a given season and month.
+// The month should be the full lowercase name, e.g., "october", "november".
+// If db is nil, it will perform a "dry run" and print the parsed data to the console.
+func FetchAndStoreGameSchedule(db *gorm.DB, season int, month string) error {
+	url := fmt.Sprintf(gameScheduleURLFmt, season, month)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to fetch schedule for %s %d: %w", month, season, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("⚠️  Skipping schedule for %s %d (Status: %s)", month, season, resp.Status)
+		return nil // Not a fatal error, just no data for this month.
+	}
+
+	htmlBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read response body for %s %d: %w", month, season, err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
+	if err != nil {
+		return fmt.Errorf("failed to parse HTML for %s %d: %w", month, season, err)
+	}
+
+	table := doc.Find("table#schedule")
+	if table.Length() == 0 {
+		// Sometimes the content is commented out
+		commentNode := doc.Find("#all_schedule").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+			return goquery.NodeName(s) == "#comment"
+		})
+		if commentNode.Length() > 0 {
+			commentedHTML := commentNode.Nodes[0].FirstChild.Data
+			innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
+			if err != nil {
+				return fmt.Errorf("failed to parse commented schedule HTML: %w", err)
+			}
+			table = innerDoc.Find("table#schedule")
+		}
+	}
+
+	if table.Length() == 0 {
+		log.Printf("No schedule table found for %s %d.", month, season)
+		return nil
+	}
+
+	var gamesToUpsert []models.Game
+	table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
+		// Skip table header rows that are sometimes repeated in the body
+		if row.Find("th.poptip").Length() > 1 {
+			return
+		}
+
+		// Skip rows that don't represent games (e.g., placeholder rows)
+		if row.Find(`[data-stat="visitor_team_name"]`).Text() == "" {
+			return
+		}
+
+		var game models.Game
+		var gameID string
+
+		// Extract GameID from the box score link, which is the most reliable unique key
+		boxScoreCell := row.Find(`td[data-stat="box_score_text"] a`)
+		if href, exists := boxScoreCell.Attr("href"); exists {
+			parts := strings.Split(href, "/")
+			fileName := parts[len(parts)-1]
+			gameID = strings.TrimSuffix(fileName, ".html")
+		}
+
+		// If there's no box score link, it's likely a future game, we can skip it or handle differently
+		if gameID == "" {
+			return
+		}
+		game.GameID = gameID
+
+		// Get the date part from the 'csk' attribute for accuracy
+		dateCsk, _ := row.Find(`th[data-stat="date_game"]`).Attr("csk")
+		var datePart string
+		if len(dateCsk) >= 8 {
+			datePart = dateCsk[:8]
+		} else {
+			log.Printf("Could not parse date from invalid 'csk' attribute: %s. Skipping.", dateCsk)
+			return
+		}
+
+		// Get the start time string
+		startTimeET := row.Find(`td[data-stat="game_start_time"]`).Text()
+		if startTimeET == "" {
+			// Skip games without a start time, as they can't be parsed accurately
+			log.Printf("Could not find start time for game %s. Skipping.", gameID)
+			return
+		}
+
+		// Load the US/Eastern timezone to correctly handle ET/EST/EDT
+		eastern, err := time.LoadLocation("America/New_York")
+		if err != nil {
+			log.Printf("FATAL: Could not load America/New_York timezone: %v", err)
+			// This is a system-level error, so we stop the row processing here.
+			// The function will continue and process any games already parsed.
+			return
+		}
+
+		// Combine date and time and parse together in the correct timezone.
+		// The layout "3:04p" handles times like "7:30p".
+		fullDateTimeString := datePart + startTimeET
+		layout := "200601023:04p"
+		gameDate, err := time.ParseInLocation(layout, fullDateTimeString, eastern)
+		if err != nil {
+			log.Printf("Could not parse combined date-time for game %s (value: '%s'): %v. Skipping.", gameID, fullDateTimeString, err)
+			return
+		}
+		game.Date = gameDate
+		game.StartTimeET = startTimeET // Keep the original string as well
+
+		game.VisitorTeam = row.Find(`td[data-stat="visitor_team_name"] a`).Text()
+		game.VisitorPTS = mustAtoi(row.Find(`td[data-stat="visitor_pts"]`).Text())
+		game.HomeTeam = row.Find(`td[data-stat="home_team_name"] a`).Text()
+		game.HomePTS = mustAtoi(row.Find(`td[data-stat="home_pts"]`).Text())
+		game.BoxScoreURL, _ = boxScoreCell.Attr("href")
+		game.GameDuration = row.Find(`td[data-stat="game_duration"]`).Text()
+		game.Arena = row.Find(`td[data-stat="arena_name"]`).Text()
+		game.IsPlayoff = strings.Contains(row.Find(`td[data-stat="game_remarks"]`).Text(), "Playoffs")
+
+		gamesToUpsert = append(gamesToUpsert, game)
+	})
+
+	if len(gamesToUpsert) > 0 {
+		// If the db connection is nil, we're in test/debug mode. Print to console.
+		if db == nil {
+			log.Println("--- RUNNING IN DRY-RUN MODE ---")
+			for _, game := range gamesToUpsert {
+				// Use %+v to print the struct with field names for clarity
+				log.Printf("Game Data: %+v\n", game)
+			}
+			log.Printf("--- WOULD INSERT %d RECORDS ---", len(gamesToUpsert))
+			return nil // End execution for dry-run
+		}
+
+		log.Printf("Attempting to batch upsert %d games for %s %d...", len(gamesToUpsert), month, season)
+
+		if err := db.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "game_id"}},
+			DoUpdates: clause.AssignmentColumns(allGameColumns()),
+		}).Create(&gamesToUpsert).Error; err != nil {
+			log.Printf("Failed to batch upsert games: %v", err)
+			return err
+		}
+		log.Printf("✅ Successfully batch upserted %d game records for %s %d.", len(gamesToUpsert), month, season)
+	} else {
+		log.Printf("No game data found to import for %s %d.", month, season)
+	}
+
+	return nil
+}
+
+// allGameColumns returns a list of all column names in the Game model for the upsert operation.
+// This ensures that if a record exists, all its fields are updated with the new data.
+func allGameColumns() []string {
+	return []string{
+		"date", "is_playoff", "start_time_et", "arena", "visitor_team",
+		"visitor_pts", "home_team", "home_pts", "game_duration", "box_score_url",
+		"updated_at",
+	}
+}

services/helpers.go

@@ -8,6 +8,16 @@ func mustAtoi(s string) int {
     return i
 }
 
+// mustAtoiWithSign handles strings that might have a "+" or "-" sign.
+func mustAtoiWithSign(s string) int {
+	if s == "" {
+		return 0
+	}
+	// The strconv.Atoi function handles the sign automatically.
+	i, _ := strconv.Atoi(s)
+	return i
+}
+
 // mustParseFloat parses s into a float64, or returns 0.0 on error.
 func mustParseFloat(s string) float64 {
     f, _ := strconv.ParseFloat(s, 64)

services/line_score_scrape_service.go

@@ -0,0 +1,54 @@
+// File: services/line_score_scrape_service.go
+package services
+
+import (
+	"log"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// FetchAndStoreLineScore parses the line score table from a goquery document and upserts it to the DB.
+// It assumes the document has already been processed to make commented-out tables visible.
+func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
+	var lineScores []models.LineScore
+
+	doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
+		teamName := row.Find(`th[data-stat="team"] a`).Text()
+		if teamName == "" {
+			return // Skip invalid rows
+		}
+
+		ls := models.LineScore{
+			GameID: gameID,
+			Team:   teamName,
+		}
+		ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
+		ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
+		ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
+		ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
+		ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
+		ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
+		ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
+		ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+		lineScores = append(lineScores, ls)
+	})
+
+	if len(lineScores) == 0 {
+		log.Printf("No line score data found to import for game %s.", gameID)
+		return nil
+	}
+
+	// Upsert the data into the database.
+	if err := db.Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+		DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+	}).Create(&lineScores).Error; err != nil {
+		return err
+	}
+	
+	log.Printf("Successfully processed line scores for game %s.", gameID)
+	return nil
+}

test/run_scraper.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// LineScore struct and mustAtoi helper function remain the same
+type LineScore struct {
+	GameID string
+	Team   string
+	Q1, Q2, Q3, Q4 int
+	OT1, OT2, OT3  int
+	Total          int
+}
+func mustAtoi(s string) int { i, _ := strconv.Atoi(s); return i }
+
+
+func scrapeAndPrintLineScore(gameURL, gameID string) {
+	// 1. Fetch the page
+	resp, err := http.Get(gameURL)
+	if err != nil {
+		log.Fatalf("Failed to fetch page: %v", err)
+	}
+	defer resp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatalf("Failed to read response body: %v", err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(bodyBytes)))
+	if err != nil {
+		log.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	var table *goquery.Selection
+	
+	// Try to find a visible table first
+	table = doc.Find("table#line_score")
+
+	if table.Length() == 0 {
+		log.Println("Visible table not found. Finding and parsing commented table...")
+		
+		commentNode := doc.Find("#all_line_score").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+			return goquery.NodeName(s) == "#comment"
+		})
+
+		if commentNode.Length() > 0 {
+			// ✅ THE FIX: Access the .Data field directly from the comment node.
+			commentedHTML := commentNode.Nodes[0].Data
+			
+			innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
+			if err != nil {
+				log.Fatalf("❌ Failed to parse the HTML from the comment: %v", err)
+			}
+			table = innerDoc.Find("table#line_score")
+		}
+	}
+
+	if table == nil || table.Length() == 0 {
+		log.Println("❌ No line score table found by any method.")
+		return
+	}
+
+	// 3. Parse the data from the found table
+	var lineScores []LineScore
+	table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
+		teamName := row.Find(`th[data-stat="team"] a`).Text()
+		if teamName != "" {
+			ls := LineScore{GameID: gameID, Team: teamName}
+			ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
+			ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
+			ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
+			ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
+			ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+			lineScores = append(lineScores, ls)
+		}
+	})
+
+	// 4. Print the results
+	if len(lineScores) > 0 {
+		log.Printf("✅ Success! Found %d line scores for game %s:", len(lineScores), gameID)
+		for _, ls := range lineScores {
+			log.Printf("%+v\n", ls)
+		}
+	} else {
+		log.Println("❌ Table was found, but failed to parse any rows.")
+	}
+}
+
+func main() {
+	gameID := "202406170BOS"
+	gameURL := fmt.Sprintf("https://www.basketball-reference.com/boxscores/%s.html", gameID)
+	log.Printf("--- Scraping Line Score for Game: %s ---", gameID)
+	log.Printf("--- URL: %s ---", gameURL)
+	scrapeAndPrintLineScore(gameURL, gameID)
+}