nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 05:55:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #42 from nprasad2077/games

Browse Source 
Games
This commit is contained in:
nprasad2077
2026-02-01 21:30:24 -06:00
committed by GitHub
parent 5f4ad9b0d6 316e786480
commit 81aa168443
15 changed files with 3833 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.DS_Store
Vendored
BIN
View File
Binary file not shown.
.gitignore
+3 -1
View File
@@ -29,4 +29,6 @@ go.work.sum
data/nba.db
/data/
/data
/data/*.DS_Store
/data/*
docs/restore_dump_remote.md
./docs/restore_dump_remote.md
Makefile
+10
View File
@@ -0,0 +1,10 @@
# Tells 'make' these are command names, not files
.PHONY: up down
# Command to run: make up
up:
docker-compose -f docker-compose.local.yml up --build -d
# Command to run: make down
down:
docker-compose down
README.md
+8 -4
View File
@@ -1,6 +1,6 @@
# NBA_Go
### First‑time bootstrap
## First‑time bootstrap
```bash
# 1. build + run
@@ -18,19 +18,23 @@ curl http://localhost:8080/api/playeradvancedstats \
```
### Swagger Initiate Docs
## Swagger Initiate Docs
```bash
swag init -g main.go -o docs
```
### Test
## Test
```bash
go run loadtest.go -n 100 -c 10 -url "http://127.0.0.1:8080/api/playeradvancedstats?page=1&pageSize=20" -log results.log -key "xxx"
```
### Local Environment
## Local Environment
```bash
docker compose down
```
```bash
docker-compose -f docker-compose.local.yml up --build -d
docker-compose.local.yml
+1 -1
View File
@@ -125,4 +125,4 @@ volumes:
grafana_data:
networks:
api.network: null
api.network: null
docs/.DS_Store
Vendored
BIN
View File
Binary file not shown.
docs/postman/NBA Statistics Go.postman_collection.json
+2505 -288
View File
File diff suppressed because one or more lines are too long
docs/postman/performance/NBA-Statistics-Go-performance-report-15.html
+1122
View File
File diff suppressed because one or more lines are too long
docs/postman/performance/restore_dump_remote.md
BIN
View File
Binary file not shown.
docs/postman/postman_docs.md
+24
View File
@@ -0,0 +1,24 @@
# Run Collection Button Code
## HTML
```html
<div class="postman-run-button"
data-postman-action="collection/fork"
data-postman-visibility="public"
data-postman-var-1="25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6"
data-postman-collection-url="entityId=25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6&entityType=collection&workspaceId=bb6fcb02-8388-4ce0-8623-63480a6dc153"></div>
<script type="text/javascript">
(function (p,o,s,t,m,a,n) {
!p[s] && (p[s] = function () { (p[t] || (p[t] = [])).push(arguments); });
!o.getElementById(s+t) && o.getElementsByTagName("head")[0].appendChild((
(n = o.createElement("script")),
(n.id = s+t), (n.async = 1), (n.src = m), n
));
}(window, document, "_pm", "PostmanRunObject", "https://run.pstmn.io/button.js"));
</script>
```
## Markdown
[<img src="https://run.pstmn.io/button.svg" alt="Run In Postman" style="width: 128px; height: 32px;">](https://god.gw.postman.com/run-collection/25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D25652688-f0cb3c13-063b-490a-aa7f-5af135aacbe6%26entityType%3Dcollection%26workspaceId%3Dbb6fcb02-8388-4ce0-8623-63480a6dc153)
docs/restore_dump_local.md
+13
View File
@@ -0,0 +1,13 @@
# Restore to local container
```bash
docker exec nba_go-postgres-1 pg_dump -U nbago -Fc nba_db > ./2025.12.05_nba_backup.dump
```
Copy file into target DB container
```bash
docker exec nba_postgres psql -U nba_admin -d nba_analytics -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
pg_restore -U nba_admin -d nba_analytics --no-owner --no-acl ./2025.12.05_nba_backup.dump
```
import.go
+38 -23
View File
@@ -3,32 +3,33 @@ package main
import (
"log"
"time"
"fmt"
"github.com/nprasad2077/NBA_Go/services"
"github.com/nprasad2077/NBA_Go/utils"
"gorm.io/gorm"
)
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
// importPlayerAdvanced fetches and stores advanced stats for seasons
func importPlayerAdvanced(db *gorm.DB) {
for season := 2026; season <= 2026; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1000 * time.Millisecond)
}
}
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2026; season <= 2026; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
log.Printf("Advanced Playoffs import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1500 * time.Millisecond)
}
}
@@ -40,7 +41,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
log.Printf("Player Totals import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1250 * time.Millisecond)
}
}
@@ -52,7 +53,7 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
log.Printf("Player Playoffs Totals import for season: %d", season)
time.Sleep(1100 * time.Millisecond)
time.Sleep(1300 * time.Millisecond)
utils.SleepWithJitter(1700 * time.Millisecond)
}
}
@@ -61,12 +62,11 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
func importGameSchedules(db *gorm.DB) {
// An NBA season typically runs from October to June
months := []string{
// "october", "november", "december", "january",
// "september", "october", "november", "december", "january",
// "february", "march", "april", "may", "june",
"february",
"january", "february",
// "february", "march", "april", "may", "june",
// "october", "november", "december",
}
for season := 2025; season <= 2026; season++ {
@@ -79,7 +79,7 @@ func importGameSchedules(db *gorm.DB) {
}
log.Printf("Game schedule import for %s, %d complete.", month, season)
// Respectful delay between requests
time.Sleep(1100 * time.Millisecond)
time.Sleep(1400 * time.Millisecond)
utils.SleepWithJitter(1800 * time.Millisecond)
}
log.Printf("--- Finished Game Schedule Import for Season: %d ---", season)
@@ -89,20 +89,27 @@ func importGameSchedules(db *gorm.DB) {
// importBoxScores fetches and stores all box score data (line scores, player/team stats)
// for games within a recent date range.
func importBoxScores(db *gorm.DB) {
// Define the date range for the 2023-2024 NBA season.
// The regular season typically starts in October and playoffs end in June.
from := time.Date(2026, time.February, 1, 0, 0, 0, 0, time.UTC)
to := time.Date(2026, time.February, 10, 0, 0, 0, 0, time.UTC)
// now := time.Now()
log.Printf("--- Starting Box Score Data Import for the 2023-2024 Season ---")
from := time.Date(2026, time.January, 27, 0, 0, 0, 0, time.UTC)
// from := time.Date(now.Year(), now.Month(), now.Day()-1, 5, 30, 0, 0, time.UTC)
to := time.Date(2026, time.February, 28, 5, 30, 0, 0, time.UTC)
dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---",
from.Format("January 2, 2006"),
to.Format("January 2, 2006"))
log.Println(dateRangeComment)
if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil {
log.Fatalf("Box score import failed: %v", err)
}
log.Printf("--- Finished Box Score Data Import ---")
log.Println("--- Finished Box Score Data Import ---")
}
// importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
func importPlayerShotCharts(db *gorm.DB) {
log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -126,14 +133,22 @@ func importPlayerShotCharts(db *gorm.DB) {
"hardeja01",
"irvinky01",
"duranke01",
// "youngtr01",
"youngtr01",
// "lillada01",
// "gilgesh01",
// "brunsja01",
// "edwaran01",
"gilgesh01",
"brunsja01",
"edwaran01",
// "mitchdo01",
// "bookede01",
// "derozde01",
"derozde01",
// "aldrila01",
"westbru01",
// "paulch01",
"butleji01",
"davisan02",
"leonaka01",
// "youngtr01",
"tatumja01",
// Add more player IDs here as needed
}
@@ -160,4 +175,4 @@ func importPlayerShotCharts(db *gorm.DB) {
}
log.Println("--- Finished Player Shot Chart Import from Predefined List ---")
}
}
main.go
+4 -5
View File
@@ -39,9 +39,9 @@ import (
"github.com/nprasad2077/NBA_Go/config"
"github.com/nprasad2077/NBA_Go/controllers"
_"github.com/nprasad2077/NBA_Go/docs"
"github.com/nprasad2077/NBA_Go/routes"
"github.com/nprasad2077/NBA_Go/utils/middleware"
_ "github.com/nprasad2077/NBA_Go/docs"
)
func main() {
@@ -53,8 +53,8 @@ func main() {
importPlayerAdvanced(db)
log.Println("🎉 Player Advanced Import completed successfully")
// importPlayerAdvancedPlayoffs(db)
// log.Println("🎉 Player Advanced Playoffs Import completed successfully")
// // importPlayerAdvancedPlayoffs(db)
// // log.Println("🎉 Player Advanced Playoffs Import completed successfully")
importPlayerTotalsScrape(db)
log.Println("🎉 Player Totals (scraped) Import completed successfully")
@@ -71,7 +71,6 @@ func main() {
// importPlayerShotCharts(db)
// log.Println("🎉 Player Shot Chart Import completed successfully 🎯")
log.Println("🏀 ALL Imports completed successfully ✅ 🙌")
return
}
@@ -132,4 +131,4 @@ func main() {
log.Println("shutting down…")
_ = app.Shutdown()
log.Println("bye")
}
}
services/box_score_scrape_service.go
+82 -26
View File
@@ -16,7 +16,8 @@ import (
)
const boxScoreURLBase = "https://www.basketball-reference.com"
const numWorkers = 5 // Number of concurrent scrapers. Adjust based on your machine and network.
const numWorkers = 2 // Number of concurrent scrapers. Adjust based on your machine and network.
const baseDelay = 2500 * time.Millisecond
// ScrapedResult holds all the parsed stats from a single game.
type ScrapedResult struct {
@@ -119,11 +120,23 @@ func FetchAndStoreBoxScoreDataForDateRange(db *gorm.DB, from, to time.Time) erro
// scrapeAndParseWorker is a worker goroutine that receives games, scrapes them, and sends back the result.
func scrapeAndParseWorker(id int, jobs <-chan models.Game, results chan<- ScrapedResult, wg *sync.WaitGroup) {
defer wg.Done()
// --- Stagger the start of each worker ---
// Calculate an offset based on the worker's ID to spread out the initial requests.
// We divide the base delay by the number of workers to get an even interval.
if numWorkers > 1 {
staggerAmount := time.Duration(int64(baseDelay) / int64(numWorkers))
initialDelay := time.Duration(id-1) * staggerAmount
log.Printf("Worker %d: Staggering start with an initial delay of %v", id, initialDelay)
time.Sleep(initialDelay)
}
for game := range jobs {
log.Printf("Worker %d: Processing game %s", id, game.GameID)
log.Printf("🐝 Worker %d: Processing game %s", id, game.GameID)
fullURL := boxScoreURLBase + game.BoxScoreURL
utils.SleepWithJitter(2300 * time.Millisecond)
utils.SleepWithJitter(baseDelay)
time.Sleep(1500 * time.Millisecond)
req, err := http.NewRequest("GET", fullURL, nil)
if err != nil {
@@ -226,9 +239,9 @@ func parseLineScore(doc *goquery.Document, gameID string) []models.LineScore {
Q2: mustAtoi(row.Find(`td[data-stat="2"]`).Text()),
Q3: mustAtoi(row.Find(`td[data-stat="3"]`).Text()),
Q4: mustAtoi(row.Find(`td[data-stat="4"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="OT1"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="OT2"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="OT3"]`).Text()),
OT1: mustAtoi(row.Find(`td[data-stat="1OT"]`).Text()),
OT2: mustAtoi(row.Find(`td[data-stat="2OT"]`).Text()),
OT3: mustAtoi(row.Find(`td[data-stat="3OT"]`).Text()),
Total: mustAtoi(row.Find(`td[data-stat="T"]`).Text()),
})
})
@@ -332,38 +345,81 @@ func parseTeamAdvStat(row *goquery.Selection, gameID, team string) models.TeamGa
}
}
func batchUpsertAll(db *gorm.DB, pbs []models.PlayerGameBasicStat, pas []models.PlayerGameAdvStat, tbs []models.TeamGameBasicStat, tas []models.TeamGameAdvStat) error {
const batchSize = 500 // 💡 A safe chunk size to stay under the limit.
// Batch upsert Player Basic Stats
if len(pbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
}).Create(&pbs).Error; err != nil {
return fmt.Errorf("failed to upsert player basic stats: %w", err)
for i := 0; i < len(pbs); i += batchSize {
end := i + batchSize
if end > len(pbs) {
end = len(pbs)
}
batch := pbs[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert player basic stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d player basic stats.", len(pbs))
}
// Batch upsert Player Advanced Stats
if len(pas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
}).Create(&pas).Error; err != nil {
return fmt.Errorf("failed to upsert player advanced stats: %w", err)
for i := 0; i < len(pas); i += batchSize {
end := i + batchSize
if end > len(pas) {
end = len(pas)
}
batch := pas[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert player advanced stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d player advanced stats.", len(pas))
}
// Batch upsert Team Basic Stats
if len(tbs) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
}).Create(&tbs).Error; err != nil {
return fmt.Errorf("failed to upsert team basic stats: %w", err)
for i := 0; i < len(tbs); i += batchSize {
end := i + batchSize
if end > len(tbs) {
end = len(tbs)
}
batch := tbs[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert team basic stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d team basic stats.", len(tbs))
}
// Batch upsert Team Advanced Stats
if len(tas) > 0 {
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
}).Create(&tas).Error; err != nil {
return fmt.Errorf("failed to upsert team advanced stats: %w", err)
for i := 0; i < len(tas); i += batchSize {
end := i + batchSize
if end > len(tas) {
end = len(tas)
}
batch := tas[i:end]
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})),
}).Create(&batch).Error; err != nil {
return fmt.Errorf("failed to upsert team advanced stats batch: %w", err)
}
}
log.Printf("Successfully upserted %d team advanced stats.", len(tas))
}
return nil
}
services/line_score_scrape_service.go
+23 -8
View File
@@ -25,14 +25,22 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
GameID: gameID,
Team: teamName,
}
// Regular Quarters - no changes here
ls.Q1 = mustAtoi(row.Find(`td[data-stat="1"]`).Text())
ls.Q2 = mustAtoi(row.Find(`td[data-stat="2"]`).Text())
ls.Q3 = mustAtoi(row.Find(`td[data-stat="3"]`).Text())
ls.Q4 = mustAtoi(row.Find(`td[data-stat="4"]`).Text())
ls.OT1 = mustAtoi(row.Find(`td[data-stat="OT1"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="OT2"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="OT3"]`).Text())
// --- FIX: Corrected selectors for Overtime periods ---
// The HTML uses "1OT", "2OT", etc., not "OT1", "OT2".
ls.OT1 = mustAtoi(row.Find(`td[data-stat="1OT"]`).Text())
ls.OT2 = mustAtoi(row.Find(`td[data-stat="2OT"]`).Text())
ls.OT3 = mustAtoi(row.Find(`td[data-stat="3OT"]`).Text())
// Total score
ls.Total = mustAtoi(row.Find(`td[data-stat="T"]`).Text())
lineScores = append(lineScores, ls)
})
@@ -42,13 +50,20 @@ func FetchAndStoreLineScore(db *gorm.DB, doc *goquery.Document, gameID string) e
}
// Upsert the data into the database.
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error; err != nil {
// Using a transaction is a good practice for atomicity.
err := db.Transaction(func(tx *gorm.DB) error {
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}},
DoUpdates: clause.AssignmentColumns([]string{"q1", "q2", "q3", "q4", "ot1", "ot2", "ot3", "total"}),
}).Create(&lineScores).Error
})
if err != nil {
log.Printf("Error upserting line scores for game %s: %v", gameID, err)
return err
}
log.Printf("Successfully processed line scores for game %s.", gameID)
return nil
}
}