nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

line score OT fix

Browse Source
This commit is contained in:
Ravi Prasad
2025-10-29 22:49:09 -05:00
parent 7ca427b386
commit d34321c345
5 changed files with 60 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/box_score_scrape_service.go
+16 -5
View File
@@ -16,8 +16,8 @@ import (
)
const boxScoreURLBase = "https://www.basketball-reference.com"
const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 3500 * time.Millisecond
const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 2500 * time.Millisecond
// ScrapedResult holds all the parsed stats from a single game.
type ScrapedResult struct {
@@ -120,6 +120,17 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro
// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
defer wg.Done()
// --- Stagger the start of each worker ---
// Calculate an offset based on the worker's ID to spread out the initial requests.
// We divide the base delay by the number of workers to get an even interval.
if numWorkers > 1 {
staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers))
initialDelay := time.Duration(id-1) * staggerAmount
log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay)
time.Sleep(initialDelay)
}
for game := range jobs {
log.Printf("Worker %d: Processing game %s", id, game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
@@ -228,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()),
Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
})
})
services/line_score_scrape_service.go
+23 -8
View File
@@ -25,14 +25,22 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
GameID: gameID,
Team: teamName,
}
// Regular Quarters - no changes here
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
// --- FIX: Corrected selectors for Overtime periods ---
// The HTML uses "1OT", "2OT", etc., not "OT1", "OT2".
ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
// Total score
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
})
@@ -42,13 +50,20 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
}
// Upsert the data into the database.
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error; err != nil {
// Using a transaction is a good practice for atomicity.
err := db.Transaction(func(tx *gorm.DB) error {
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error
})
if err != nil {
log.Printf("Error upserting line scores for game %s: %v", gameID, err)
return err
}
log.Printf("Successfully processed line scores for game %s.", gameID)
return nil
}
}