mirror of
https://github.com/nprasad2077/NBA_Go.git
synced 2026-09-22 22:15:13 +00:00
242 lines
8.4 KiB
Go
242 lines
8.4 KiB
Go
// File: services/game_scrape_service.go
|
|
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/nprasad2077/NBA_Go/models"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
)
|
|
|
|
const gameScheduleURLFmt = "https://www.basketball-reference.com/leagues/NBA_%d_games-%s.html"
|
|
const playoffScheduleURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_games.html"
|
|
|
|
// FetchAndMarkPlayoffGames scrapes the dedicated playoff schedule page and
|
|
// batch-updates all matching games in the DB to set IsPlayoff=true.
|
|
// This is a post-processing step that does not modify the normal import flow.
|
|
func FetchAndMarkPlayoffGames(db *gorm.DB, season int) error {
|
|
url := fmt.Sprintf(playoffScheduleURLFmt, season)
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch playoff schedule for %d: %w", season, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("playoff schedule returned status %s for season %d", resp.Status, season)
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse playoff schedule HTML: %w", err)
|
|
}
|
|
|
|
var gameIDs []string
|
|
doc.Find(`td[data-stat="box_score_text"] a`).Each(func(i int, s *goquery.Selection) {
|
|
if href, exists := s.Attr("href"); exists {
|
|
parts := strings.Split(href, "/")
|
|
fileName := parts[len(parts)-1]
|
|
gameID := strings.TrimSuffix(fileName, ".html")
|
|
if gameID != "" {
|
|
gameIDs = append(gameIDs, gameID)
|
|
}
|
|
}
|
|
})
|
|
|
|
if len(gameIDs) == 0 {
|
|
log.Printf("No playoff game IDs found for season %d.", season)
|
|
return nil
|
|
}
|
|
|
|
if err := db.Model(&models.Game{}).Where("game_id IN ?", gameIDs).Update("is_playoff", true).Error; err != nil {
|
|
return fmt.Errorf("failed to mark playoff games: %w", err)
|
|
}
|
|
|
|
log.Printf("✅ Marked %d games as playoff for season %d.", len(gameIDs), season)
|
|
return nil
|
|
}
|
|
|
|
// FetchAndStoreGameSchedule scrapes the game schedule for a given season and month.
|
|
// The month should be the full lowercase name, e.g., "october", "november".
|
|
// If db is nil, it will perform a "dry run" and print the parsed data to the console.
|
|
func FetchAndStoreGameSchedule(db *gorm.DB, season int, month string) error {
|
|
url := fmt.Sprintf(gameScheduleURLFmt, season, month)
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch schedule for %s %d: %w", month, season, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
log.Printf("⚠️ Skipping schedule for %s %d (Status: %s)", month, season, resp.Status)
|
|
return nil // Not a fatal error, just no data for this month.
|
|
}
|
|
|
|
htmlBytes, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read response body for %s %d: %w", month, season, err)
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse HTML for %s %d: %w", month, season, err)
|
|
}
|
|
|
|
table := doc.Find("table#schedule")
|
|
if table.Length() == 0 {
|
|
// Sometimes the content is commented out
|
|
commentNode := doc.Find("#all_schedule").Contents().FilterFunction(func(i int, s *goquery.Selection) bool {
|
|
return goquery.NodeName(s) == "#comment"
|
|
})
|
|
if commentNode.Length() > 0 {
|
|
commentedHTML := commentNode.Nodes[0].FirstChild.Data
|
|
innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse commented schedule HTML: %w", err)
|
|
}
|
|
table = innerDoc.Find("table#schedule")
|
|
}
|
|
}
|
|
|
|
if table.Length() == 0 {
|
|
log.Printf("No schedule table found for %s %d.", month, season)
|
|
return nil
|
|
}
|
|
|
|
var gamesToUpsert []models.Game
|
|
table.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
|
|
// Skip table header rows that are sometimes repeated in the body
|
|
if row.Find("th.poptip").Length() > 1 {
|
|
return
|
|
}
|
|
|
|
// Skip rows that don't represent games (e.g., placeholder rows)
|
|
if row.Find(`[data-stat="visitor_team_name"]`).Text() == "" {
|
|
return
|
|
}
|
|
|
|
var game models.Game
|
|
var gameID string
|
|
|
|
// Extract GameID from the box score link, which is the most reliable unique key
|
|
boxScoreCell := row.Find(`td[data-stat="box_score_text"] a`)
|
|
if href, exists := boxScoreCell.Attr("href"); exists {
|
|
parts := strings.Split(href, "/")
|
|
fileName := parts[len(parts)-1]
|
|
gameID = strings.TrimSuffix(fileName, ".html")
|
|
}
|
|
|
|
// If there's no box score link, it's likely a future game, we can skip it or handle differently
|
|
if gameID == "" {
|
|
return
|
|
}
|
|
game.GameID = gameID
|
|
|
|
// Get the date part from the 'csk' attribute for accuracy
|
|
dateCsk, _ := row.Find(`th[data-stat="date_game"]`).Attr("csk")
|
|
var datePart string
|
|
if len(dateCsk) >= 8 {
|
|
datePart = dateCsk[:8]
|
|
} else {
|
|
log.Printf("Could not parse date from invalid 'csk' attribute: %s. Skipping.", dateCsk)
|
|
return
|
|
}
|
|
|
|
// Get the start time string
|
|
startTimeET := row.Find(`td[data-stat="game_start_time"]`).Text()
|
|
if startTimeET == "" {
|
|
// Skip games without a start time, as they can't be parsed accurately
|
|
log.Printf("Could not find start time for game %s. Skipping.", gameID)
|
|
return
|
|
}
|
|
|
|
// Load the US/Eastern timezone to correctly handle ET/EST/EDT
|
|
eastern, err := time.LoadLocation("America/New_York")
|
|
if err != nil {
|
|
log.Printf("FATAL: Could not load America/New_York timezone: %v", err)
|
|
// This is a system-level error, so we stop the row processing here.
|
|
// The function will continue and process any games already parsed.
|
|
return
|
|
}
|
|
|
|
// Combine date and time and parse together in the correct timezone.
|
|
// The layout "3:04p" handles times like "7:30p".
|
|
fullDateTimeString := datePart + startTimeET
|
|
layout := "200601023:04p"
|
|
gameDate, err := time.ParseInLocation(layout, fullDateTimeString, eastern)
|
|
if err != nil {
|
|
log.Printf("Could not parse combined date-time for game %s (value: '%s'): %v. Skipping.", gameID, fullDateTimeString, err)
|
|
return
|
|
}
|
|
game.Date = gameDate
|
|
game.StartTimeET = startTimeET // Keep the original string as well
|
|
|
|
game.VisitorTeam = row.Find(`td[data-stat="visitor_team_name"] a`).Text()
|
|
game.VisitorPTS = mustAtoi(row.Find(`td[data-stat="visitor_pts"]`).Text())
|
|
game.HomeTeam = row.Find(`td[data-stat="home_team_name"] a`).Text()
|
|
game.HomePTS = mustAtoi(row.Find(`td[data-stat="home_pts"]`).Text())
|
|
game.BoxScoreURL, _ = boxScoreCell.Attr("href")
|
|
game.GameDuration = row.Find(`td[data-stat="game_duration"]`).Text()
|
|
game.Arena = row.Find(`td[data-stat="arena_name"]`).Text()
|
|
game.IsPlayoff = strings.Contains(row.Find(`td[data-stat="game_remarks"]`).Text(), "Playoffs")
|
|
|
|
gamesToUpsert = append(gamesToUpsert, game)
|
|
})
|
|
|
|
if len(gamesToUpsert) > 0 {
|
|
// If the db connection is nil, we're in test/debug mode. Print to console.
|
|
if db == nil {
|
|
log.Println("--- RUNNING IN DRY-RUN MODE ---")
|
|
for _, game := range gamesToUpsert {
|
|
// Use %+v to print the struct with field names for clarity
|
|
log.Printf("Game Data: %+v\n", game)
|
|
}
|
|
log.Printf("--- WOULD INSERT %d RECORDS ---", len(gamesToUpsert))
|
|
return nil // End execution for dry-run
|
|
}
|
|
|
|
log.Printf("Attempting to batch upsert %d games for %s %d...", len(gamesToUpsert), month, season)
|
|
|
|
if err := db.Clauses(clause.OnConflict{
|
|
Columns: []clause.Column{{Name: "game_id"}},
|
|
DoUpdates: clause.AssignmentColumns(allGameColumns()),
|
|
}).Create(&gamesToUpsert).Error; err != nil {
|
|
log.Printf("Failed to batch upsert games: %v", err)
|
|
return err
|
|
}
|
|
log.Printf("✅ Successfully batch upserted %d game records for %s %d.", len(gamesToUpsert), month, season)
|
|
} else {
|
|
log.Printf("No game data found to import for %s %d.", month, season)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// allGameColumns returns a list of all column names in the Game model for the upsert operation.
|
|
// This ensures that if a record exists, all its fields are updated with the new data.
|
|
func allGameColumns() []string {
|
|
return []string{
|
|
"date", "is_playoff", "start_time_et", "arena", "visitor_team",
|
|
"visitor_pts", "home_team", "home_pts", "game_duration", "box_score_url",
|
|
"updated_at",
|
|
}
|
|
}
|