nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 14:05:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

line score OT fix

Browse Source
This commit is contained in:
Ravi Prasad
2025-10-29 22:49:09 -05:00
parent 7ca427b386
commit d34321c345
5 changed files with 60 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/box_score_scrape_service.go
+16 -5
View File
@@ -16,8 +16,8 @@ import (
)
const boxScoreURLBase = "https://www.basketball-reference.com"
const numWorkers = 1 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 3500 * time.Millisecond
const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 2500 * time.Millisecond
// ScrapedResult holds all the parsed stats from a single game.
type ScrapedResult struct {
@@ -120,6 +120,17 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro
// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
defer wg.Done()
// --- Stagger the start of each worker ---
// Calculate an offset based on the worker's ID to spread out the initial requests.
// We divide the base delay by the number of workers to get an even interval.
if numWorkers > 1 {
staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers))
initialDelay := time.Duration(id-1) * staggerAmount
log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay)
time.Sleep(initialDelay)
}
for game := range jobs {
log.Printf("Worker %d: Processing game %s", id, game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
@@ -228,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()),
Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
})
})