nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 05:55:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

advanced scraping usage works

Browse Source
This commit is contained in:
Ravi Prasad
2025-05-29 00:29:00 -05:00
parent cdf5cf4cd2
commit 47ea78901a
3 changed files with 32 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
import.go
+5 -5
View File
@@ -11,7 +11,7 @@ import (
// importPlayerAdvanced fetches and stores advanced stats for seasons 2023–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2023; season <= 2025; season++ {
if err := services.FetchAndStorePlayerAdvancedStats(db, season, false); err != nil {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
time.Sleep(1100 * time.Millisecond)
@@ -21,8 +21,8 @@ func importPlayerAdvanced(db *gorm.DB) {
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2023; season <= 2024; season++ {
if err := services.FetchAndStorePlayerAdvancedPlayoffsStats(db, season, true); err != nil {
for season := 2023; season <= 2025; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
time.Sleep(1100 * time.Millisecond)
@@ -70,8 +70,8 @@ func importPlayerTotalsScrape(db *gorm.DB) {
}
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerPlayoffsScrape(db *gorm.DB) {
for season := 2023; season <= 2024; season++ {
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 2023; season <= 2025; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
main.go
+4 -1
View File
@@ -26,6 +26,7 @@ import (
fiberswagger "github.com/swaggo/fiber-swagger"
"github.com/nprasad2077/NBA_Go/config"
"github.com/nprasad2077/NBA_Go/utils"
"github.com/nprasad2077/NBA_Go/controllers"
"github.com/nprasad2077/NBA_Go/routes"
"github.com/nprasad2077/NBA_Go/utils/middleware"
@@ -39,6 +40,8 @@ func main() {
importPlayerAdvanced(db)
log.Println("🎉 Player Advanced Import completed successfully")
utils.SleepWithJitter(1100 * time.Millisecond)
importPlayerAdvancedPlayoffs(db)
log.Println("🎉 Player Advanced Playoffs Import completed successfully")
@@ -49,7 +52,7 @@ func main() {
importPlayerTotalsScrape(db)
log.Println("🎉 Player Totals (scraped) Import completed successfully")
importPlayerPlayoffsScrape(db)
importPlayerTotalsPlayoffsScrape(db)
log.Println("🎉 Player Playoffs (scraped) Import completed successfully")
// importPlayerShotChart(db)
utils/sleep.go
+23
View File
@@ -0,0 +1,23 @@
// File: utils/sleep.go
package utils
import (
"math/rand"
"time"
)
func init() {
// Seed the RNG once when the package is initialized
rand.Seed(time.Now().UnixNano())
}
// SleepWithJitter sleeps for the given base duration plus or minus 25% random jitter.
// This helps to avoid perfectly uniform request intervals.
func SleepWithJitter(base time.Duration) {
half := base / 2
// Random value in [0, half)
randOffset := time.Duration(rand.Int63n(int64(half)))
// Shift to center jitter around zero: [-half/2, +half/2)
delta := randOffset - half/2
time.Sleep(base + delta)
}