line score service isolated and working

models/game.go

@@ -34,8 +34,8 @@ type Game struct {
 // LineScore holds the scoring for each team by quarter for a specific game.
 type LineScore struct {
 	ID     uint   `gorm:"primaryKey" swaggerignore:"true"`
-	GameID string `gorm:"not null;index" json:"gameId"`
+	GameID string `gorm:"not null;uniqueIndex:idx_line_score,priority:1" json:"gameId"`
-	Team   string `gorm:"not null;index" json:"team"`
+	Team   string `gorm:"not null;uniqueIndex:idx_line_score,priority:2" json:"team"`
 	Q1     int    `json:"q1"`
 	Q2     int    `json:"q2"`
 	Q3     int    `json:"q3"`

services/box_score_scrape_service.go

@@ -1,4 +1,3 @@
-// File: services/box_score_scrape_service.go
 package services
 
 import (
@@ -17,6 +16,21 @@ import (
 
 const boxScoreURLBase = "https://www.basketball-reference.com"
 
+// uncommentDoc finds and replaces commented out HTML sections.
+func uncommentDoc(doc *goquery.Document) *goquery.Document {
+	doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+		return goquery.NodeName(s) == "#comment"
+	}).Each(func(i int, s *goquery.Selection) {
+		// Use .Data on the underlying html.Node to get the comment content.
+		commentText := s.Nodes[0].Data
+		if strings.Contains(commentText, "<table") {
+			// Replace the comment node with its content.
+			s.ReplaceWithHtml(commentText)
+		}
+	})
+	return doc
+}
+
 // FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
 func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
 	var games []models.Game
@@ -62,13 +76,15 @@ func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
 		return err
 	}
 
-	// The content is often inside a comment, so we need to extract it.
+	// 1. Uncomment all tables in the document first.
 	doc = uncommentDoc(doc)
 
-	// --- Scrape all data types from the page ---
+	// 2. Call the dedicated service to handle line scores.
-	if err := parseAndStoreLineScore(db, doc, gameID); err != nil {
+	if err := FetchAndStoreLineScore(db, doc, gameID); err != nil {
-		return err
+		log.Printf("Error processing line score for game %s: %v", gameID, err)
 	}
+
+	// 3. The existing box score parser will now work because its tables are visible.
 	if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
 		return err
 	}
@@ -76,33 +92,6 @@ func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
 	return nil
 }
 
-// parseAndStoreLineScore scrapes the line score table.
-func parseAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
-	var lineScores []models.LineScore
-	doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
-		ls := models.LineScore{GameID: gameID}
-		ls.Team = row.Find(`th[data-stat="team"] a`).Text()
-		ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
-		ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
-		ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
-		ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
-		// FIX: Corrected the data-stat attribute for overtime periods.
-		ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
-		ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
-		ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
-		ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
-		lineScores = append(lineScores, ls)
-	})
-
-	if len(lineScores) > 0 {
-		return db.Clauses(clause.OnConflict{
-			Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
-			DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
-		}).Create(&lineScores).Error
-	}
-	return nil
-}
-
 // parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
 func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
 	var allPlayerBasicStats []models.PlayerGameBasicStat
@@ -304,19 +293,7 @@ func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.
 	return nil
 }
 
-// uncommentDoc finds and replaces commented out HTML sections.
+
-func uncommentDoc(doc *goquery.Document) *goquery.Document {
-	doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) {
-		s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool {
-			return goquery.NodeName(s) == "#comment"
-		}).Each(func(i int, s *goquery.Selection) {
-			uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text()))
-			// FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string.
-			s.ReplaceWithSelection(uncommented.Find("body").Children())
-		})
-	})
-	return doc
-}
 
 // getModelColumns is a placeholder for a more robust reflection-based column name generator.
 // For now, it returns hardcoded lists.

services/line_score_scrape_service.go

@@ -0,0 +1,54 @@
+// File: services/line_score_scrape_service.go
+package services
+
+import (
+	"log"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/nprasad2077/NBA_Go/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// FetchAndStoreLineScore parses the line score table from a goquery document and upserts it to the DB.
+// It assumes the document has already been processed to make commented-out tables visible.
+func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
+	var lineScores []models.LineScore
+
+	doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
+		teamName := row.Find(`th[data-stat="team"] a`).Text()
+		if teamName == "" {
+			return // Skip invalid rows
+		}
+
+		ls := models.LineScore{
+			GameID: gameID,
+			Team:   teamName,
+		}
+		ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
+		ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
+		ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
+		ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
+		ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
+		ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
+		ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
+		ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+		lineScores = append(lineScores, ls)
+	})
+
+	if len(lineScores) == 0 {
+		log.Printf("No line score data found to import for game %s.", gameID)
+		return nil
+	}
+
+	// Upsert the data into the database.
+	if err := db.Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "game_id"}, {Name: "team"}},
+		DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
+	}).Create(&lineScores).Error; err != nil {
+		return err
+	}
+	
+	log.Printf("Successfully processed line scores for game %s.", gameID)
+	return nil
+}

test/run_scraper.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// LineScore struct and mustAtoi helper function remain the same
+type LineScore struct {
+	GameID string
+	Team   string
+	Q1, Q2, Q3, Q4 int
+	OT1, OT2, OT3  int
+	Total          int
+}
+func mustAtoi(s string) int { i, _ := strconv.Atoi(s); return i }
+
+
+func scrapeAndPrintLineScore(gameURL, gameID string) {
+	// 1. Fetch the page
+	resp, err := http.Get(gameURL)
+	if err != nil {
+		log.Fatalf("Failed to fetch page: %v", err)
+	}
+	defer resp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatalf("Failed to read response body: %v", err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(bodyBytes)))
+	if err != nil {
+		log.Fatalf("Failed to parse HTML: %v", err)
+	}
+
+	var table *goquery.Selection
+	
+	// Try to find a visible table first
+	table = doc.Find("table#line_score")
+
+	if table.Length() == 0 {
+		log.Println("Visible table not found. Finding and parsing commented table...")
+		
+		commentNode := doc.Find("#all_line_score").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
+			return goquery.NodeName(s) == "#comment"
+		})
+
+		if commentNode.Length() > 0 {
+			// ✅ THE FIX: Access the .Data field directly from the comment node.
+			commentedHTML := commentNode.Nodes[0].Data
+			
+			innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
+			if err != nil {
+				log.Fatalf("❌ Failed to parse the HTML from the comment: %v", err)
+			}
+			table = innerDoc.Find("table#line_score")
+		}
+	}
+
+	if table == nil || table.Length() == 0 {
+		log.Println("❌ No line score table found by any method.")
+		return
+	}
+
+	// 3. Parse the data from the found table
+	var lineScores []LineScore
+	table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
+		teamName := row.Find(`th[data-stat="team"] a`).Text()
+		if teamName != "" {
+			ls := LineScore{GameID: gameID, Team: teamName}
+			ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
+			ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
+			ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
+			ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
+			ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
+			lineScores = append(lineScores, ls)
+		}
+	})
+
+	// 4. Print the results
+	if len(lineScores) > 0 {
+		log.Printf("✅ Success! Found %d line scores for game %s:", len(lineScores), gameID)
+		for _, ls := range lineScores {
+			log.Printf("%+v\n", ls)
+		}
+	} else {
+		log.Println("❌ Table was found, but failed to parse any rows.")
+	}
+}
+
+func main() {
+	gameID := "202406170BOS"
+	gameURL := fmt.Sprintf("https://www.basketball-reference.com/boxscores/%s.html", gameID)
+	log.Printf("--- Scraping Line Score for Game: %s ---", gameID)
+	log.Printf("--- URL: %s ---", gameURL)
+	scrapeAndPrintLineScore(gameURL, gameID)
+}