nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 05:55:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

line score service isolated and working

Browse Source
This commit is contained in:
Ravi Prasad
2025-07-08 20:20:39 -05:00
parent 5f7a1eb355
commit fe1379e54e
4 changed files with 182 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/game.go
+2 -2
View File
@@ -34,8 +34,8 @@ type Game struct {
// LineScore holds the scoring for each team by quarter for a specific game.
type LineScore struct {
ID uint `gorm:"primaryKey" swaggerignore:"true"`
GameID string `gorm:"not null;index" json:"gameId"`
Team string `gorm:"not null;index" json:"team"`
GameID string `gorm:"not null;uniqueIndex:idx_line_score,priority:1" json:"gameId"`
Team string `gorm:"not null;uniqueIndex:idx_line_score,priority:2" json:"team"`
Q1 int `json:"q1"`
Q2 int `json:"q2"`
Q3 int `json:"q3"`
services/box_score_scrape_service.go
+22 -45
View File
@@ -1,4 +1,3 @@
// File: services/box_score_scrape_service.go
package services
import (
@@ -17,6 +16,21 @@ import (
const boxScoreURLBase = "https://www.basketball-reference.com"
// uncommentDoc finds and replaces commented out HTML sections.
func uncommentDoc(doc *goquery.Document) *goquery.Document {
doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
return goquery.NodeName(s) == "#comment"
}).Each(func(i int, s *goquery.Selection) {
// Use .Data on the underlying html.Node to get the comment content.
commentText := s.Nodes[0].Data
if strings.Contains(commentText, "<table") {
// Replace the comment node with its content.
s.ReplaceWithHtml(commentText)
}
})
return doc
}
// FetchAndStoreBoxScoreDataForDateRange fetches all games in a date range and scrapes their box scores.
func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) error {
var games []models.Game
@@ -62,13 +76,15 @@ func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
return err
}
// The content is often inside a comment, so we need to extract it.
// 1. Uncomment all tables in the document first.
doc = uncommentDoc(doc)
// --- Scrape all data types from the page ---
if err := parseAndStoreLineScore(db, doc, gameID); err != nil {
return err
// 2. Call the dedicated service to handle line scores.
if err := FetchAndStoreLineScore(db, doc, gameID); err != nil {
log.Printf("Error processing line score for game %s: %v", gameID, err)
}
// 3. The existing box score parser will now work because its tables are visible.
if err := parseAndStoreBoxScores(db, doc, gameID); err != nil {
return err
}
@@ -76,33 +92,6 @@ func scrapeBoxScorePage(db *gorm.DB, url, gameID string) error {
return nil
}
// parseAndStoreLineScore scrapes the line score table.
func parseAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
var lineScores []models.LineScore
doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
ls := models.LineScore{GameID: gameID}
ls.Team = row.Find(`th[data-stat="team"] a`).Text()
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
// FIX: Corrected the data-stat attribute for overtime periods.
ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
})
if len(lineScores) > 0 {
return db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error
}
return nil
}
// parseAndStoreBoxScores finds all basic and advanced box score tables and processes them.
func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error {
var allPlayerBasicStats []models.PlayerGameBasicStat
@@ -304,19 +293,7 @@ func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.
return nil
}
// uncommentDoc finds and replaces commented out HTML sections.
func uncommentDoc(doc *goquery.Document) *goquery.Document {
doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) {
s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool {
return goquery.NodeName(s) == "#comment"
}).Each(func(i int, s *goquery.Selection) {
uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text()))
// FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string.
s.ReplaceWithSelection(uncommented.Find("body").Children())
})
})
return doc
}
// getModelColumns is a placeholder for a more robust reflection-based column name generator.
// For now, it returns hardcoded lists.
services/line_score_scrape_service.go
+54
View File
@@ -0,0 +1,54 @@
// File: services/line_score_scrape_service.go
package services
import (
"log"
"github.com/PuerkitoBio/goquery"
"github.com/nprasad2077/NBA_Go/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// FetchAndStoreLineScore parses the line score table from a goquery document and upserts it to the DB.
// It assumes the document has already been processed to make commented-out tables visible.
func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error {
var lineScores []models.LineScore
doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) {
teamName := row.Find(`th[data-stat="team"] a`).Text()
if teamName == "" {
return // Skip invalid rows
}
ls := models.LineScore{
GameID: gameID,
Team: teamName,
}
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
})
if len(lineScores) == 0 {
log.Printf("No line score data found to import for game %s.", gameID)
return nil
}
// Upsert the data into the database.
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error; err != nil {
return err
}
log.Printf("Successfully processed line scores for game %s.", gameID)
return nil
}
test/run_scraper.go
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"fmt"
"io"
"log"
"net/http"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
)
// LineScore struct and mustAtoi helper function remain the same
type LineScore struct {
GameID string
Team string
Q1, Q2, Q3, Q4 int
OT1, OT2, OT3 int
Total int
}
func mustAtoi(s string) int { i, _ := strconv.Atoi(s); return i }
func scrapeAndPrintLineScore(gameURL, gameID string) {
// 1. Fetch the page
resp, err := http.Get(gameURL)
if err != nil {
log.Fatalf("Failed to fetch page: %v", err)
}
defer resp.Body.Close()
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
log.Fatalf("Failed to read response body: %v", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(bodyBytes)))
if err != nil {
log.Fatalf("Failed to parse HTML: %v", err)
}
var table *goquery.Selection
// Try to find a visible table first
table = doc.Find("table#line_score")
if table.Length() == 0 {
log.Println("Visible table not found. Finding and parsing commented table...")
commentNode := doc.Find("#all_line_score").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
return goquery.NodeName(s) == "#comment"
})
if commentNode.Length() > 0 {
// ✅ THE FIX: Access the .Data field directly from the comment node.
commentedHTML := commentNode.Nodes[0].Data
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
if err != nil {
log.Fatalf("❌ Failed to parse the HTML from the comment: %v", err)
}
table = innerDoc.Find("table#line_score")
}
}
if table == nil || table.Length() == 0 {
log.Println("❌ No line score table found by any method.")
return
}
// 3. Parse the data from the found table
var lineScores []LineScore
table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
teamName := row.Find(`th[data-stat="team"] a`).Text()
if teamName != "" {
ls := LineScore{GameID: gameID, Team: teamName}
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
}
})
// 4. Print the results
if len(lineScores) > 0 {
log.Printf("✅ Success! Found %d line scores for game %s:", len(lineScores), gameID)
for _, ls := range lineScores {
log.Printf("%+v\n", ls)
}
} else {
log.Println("❌ Table was found, but failed to parse any rows.")
}
}
func main() {
gameID := "202406170BOS"
gameURL := fmt.Sprintf("https://www.basketball-reference.com/boxscores/%s.html", gameID)
log.Printf("--- Scraping Line Score for Game: %s ---", gameID)
log.Printf("--- URL: %s ---", gameURL)
scrapeAndPrintLineScore(gameURL, gameID)
}