diff --git a/models/game.go b/models/game.go index 6ca8ff7..79261f7 100644 --- a/models/game.go +++ b/models/game.go @@ -34,8 +34,8 @@ type Game struct { // LineScore holds the scoring for each team by quarter for a specific game. type LineScore struct { ID uint `gorm:"primaryKey" swaggerignore:"true"` - GameID string `gorm:"not null;index" json:"gameId"` - Team string `gorm:"not null;index" json:"team"` + GameID string `gorm:"not null;uniqueIndex:idx_line_score,priority:1" json:"gameId"` + Team string `gorm:"not null;uniqueIndex:idx_line_score,priority:2" json:"team"` Q1 int `json:"q1"` Q2 int `json:"q2"` Q3 int `json:"q3"` diff --git a/services/box_score_scrape_service.go b/services/box_score_scrape_service.go index 96c66f0..1f098df 100644 --- a/services/box_score_scrape_service.go +++ b/services/box_score_scrape_service.go @@ -1,4 +1,3 @@ -// File: services/box_score_scrape_service.go package services import ( @@ -17,6 +16,21 @@ import ( const boxScoreURLBase = "https://www.basketball-reference.com" +// uncommentDoc finds and replaces commented out HTML sections. +func uncommentDoc(doc *goquery.Document) *goquery.Document { + doc.Find("*").Contents().FilterFunction(func(i int, s *goquery.Selection) bool { + return goquery.NodeName(s) == "#comment" + }).Each(func(i int, s *goquery.Selection) { + // Use .Data on the underlying html.Node to get the comment content. + commentText := s.Nodes[0].Data + if strings.Contains(commentText, " 0 { - return db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}), - }).Create(&lineScores).Error - } - return nil -} - // parseAndStoreBoxScores finds all basic and advanced box score tables and processes them. func parseAndStoreBoxScores(db *gorm.DB, doc *goquery.Document, gameID string) error { var allPlayerBasicStats []models.PlayerGameBasicStat @@ -304,19 +293,7 @@ func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models. return nil } -// uncommentDoc finds and replaces commented out HTML sections. -func uncommentDoc(doc *goquery.Document) *goquery.Document { - doc.Find("#content").Find(".placeholder, .section_heading").Each(func(i int, s *goquery.Selection) { - s.NextUntil(".placeholder, .section_heading").FilterFunction(func(i int, s *goquery.Selection) bool { - return goquery.NodeName(s) == "#comment" - }).Each(func(i int, s *goquery.Selection) { - uncommented, _ := goquery.NewDocumentFromReader(strings.NewReader(s.Text())) - // FIX: Use ReplaceWithSelection, as the argument is a goquery selection, not a string. - s.ReplaceWithSelection(uncommented.Find("body").Children()) - }) - }) - return doc -} + // getModelColumns is a placeholder for a more robust reflection-based column name generator. // For now, it returns hardcoded lists. diff --git a/services/line_score_scrape_service.go b/services/line_score_scrape_service.go new file mode 100644 index 0000000..7f6f4a4 --- /dev/null +++ b/services/line_score_scrape_service.go @@ -0,0 +1,54 @@ +// File: services/line_score_scrape_service.go +package services + +import ( + "log" + + "github.com/PuerkitoBio/goquery" + "github.com/nprasad2077/NBA_Go/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// FetchAndStoreLineScore parses the line score table from a goquery document and upserts it to the DB. +// It assumes the document has already been processed to make commented-out tables visible. +func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) error { + var lineScores []models.LineScore + + doc.Find("table#line_score tbody tr").Each(func(i int, row *goquery.Selection) { + teamName := row.Find(`th[data-stat="team"] a`).Text() + if teamName == "" { + return // Skip invalid rows + } + + ls := models.LineScore{ + GameID: gameID, + Team: teamName, + } + ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text()) + ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text()) + ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text()) + ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text()) + ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()) + ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()) + ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()) + ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text()) + lineScores = append(lineScores, ls) + }) + + if len(lineScores) == 0 { + log.Printf("No line score data found to import for game %s.", gameID) + return nil + } + + // Upsert the data into the database. + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}), + }).Create(&lineScores).Error; err != nil { + return err + } + + log.Printf("Successfully processed line scores for game %s.", gameID) + return nil +} \ No newline at end of file diff --git a/test/run_scraper.go b/test/run_scraper.go new file mode 100644 index 0000000..5238f73 --- /dev/null +++ b/test/run_scraper.go @@ -0,0 +1,104 @@ +package main + +import ( + "fmt" + "io" + "log" + "net/http" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +// LineScore struct and mustAtoi helper function remain the same +type LineScore struct { + GameID string + Team string + Q1, Q2, Q3, Q4 int + OT1, OT2, OT3 int + Total int +} +func mustAtoi(s string) int { i, _ := strconv.Atoi(s); return i } + + +func scrapeAndPrintLineScore(gameURL, gameID string) { + // 1. Fetch the page + resp, err := http.Get(gameURL) + if err != nil { + log.Fatalf("Failed to fetch page: %v", err) + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatalf("Failed to read response body: %v", err) + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(bodyBytes))) + if err != nil { + log.Fatalf("Failed to parse HTML: %v", err) + } + + var table *goquery.Selection + + // Try to find a visible table first + table = doc.Find("table#line_score") + + if table.Length() == 0 { + log.Println("Visible table not found. Finding and parsing commented table...") + + commentNode := doc.Find("#all_line_score").Contents().FilterFunction(func(i int, s *goquery.Selection) bool { + return goquery.NodeName(s) == "#comment" + }) + + if commentNode.Length() > 0 { + // ✅ THE FIX: Access the .Data field directly from the comment node. + commentedHTML := commentNode.Nodes[0].Data + + innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) + if err != nil { + log.Fatalf("❌ Failed to parse the HTML from the comment: %v", err) + } + table = innerDoc.Find("table#line_score") + } + } + + if table == nil || table.Length() == 0 { + log.Println("❌ No line score table found by any method.") + return + } + + // 3. Parse the data from the found table + var lineScores []LineScore + table.Find("tbody tr").Each(func(i int, row *goquery.Selection) { + teamName := row.Find(`th[data-stat="team"] a`).Text() + if teamName != "" { + ls := LineScore{GameID: gameID, Team: teamName} + ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text()) + ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text()) + ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text()) + ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text()) + ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text()) + lineScores = append(lineScores, ls) + } + }) + + // 4. Print the results + if len(lineScores) > 0 { + log.Printf("✅ Success! Found %d line scores for game %s:", len(lineScores), gameID) + for _, ls := range lineScores { + log.Printf("%+v\n", ls) + } + } else { + log.Println("❌ Table was found, but failed to parse any rows.") + } +} + +func main() { + gameID := "202406170BOS" + gameURL := fmt.Sprintf("https://www.basketball-reference.com/boxscores/%s.html", gameID) + log.Printf("--- Scraping Line Score for Game: %s ---", gameID) + log.Printf("--- URL: %s ---", gameURL) + scrapeAndPrintLineScore(gameURL, gameID) +} \ No newline at end of file