auto-fix missing games

2026-09-22 14:05:13 +00:00 · 2026-09-10 03:24:36 -05:00
parent d32a88db89
commit d077400059
4 changed files with 190 additions and 22 deletions
@@ -160,18 +160,18 @@ Services will be available at:

 ### Importing Data

-The application has a dual-mode entry point. To run data imports (migrations + scraping):
+The application provides a dual-mode entry point. To run data migrations and imports:

-#### 1. Local CLI Execution (Recommended for targeted imports)
+#### 1. Local CLI Execution (Recommended for automated & targeted imports)
 ```bash
-# Export environment variables from .env
+# Export environment variables from .env (e.g. remote or local database)
 export $(grep -v '^#' .env | xargs)

 # Run the import pipeline
 go run . import-data
 ```

-#### 2. Local Docker Stack
+#### 2. Local Docker Development Stack
 ```bash
 docker-compose -f docker-compose.local.yml run --rm db-init
 ```
@@ -185,13 +185,52 @@ docker compose --profile init run --rm db-init

 ### Ingestion Pipeline & Scraping Architecture

-The data import engine (`import.go` & `services/`) features a robust, resilient ingestion workflow designed to safely handle thousands of games:
+The data import engine (`import.go` & `services/`) features a resilient, multi-stage ingestion workflow designed to safely scrape and persist NBA statistics without rate limits or data loss:

- **20-Day Temporal Session Chunks**: Large date ranges (such as full seasons) are automatically partitioned into 20-day sliding windows (~80–120 games per chunk).
- **Immediate Incremental Persistence**: Scraped data (`line_scores`, `player_game_basic_stats`, `player_game_adv_stats`, `team_game_basic_stats`, `team_game_adv_stats`) is immediately committed and upserted into PostgreSQL at the end of each chunk rather than held in memory until the end of the run.
- **Inter-Chunk Cool-Off Period**: Enforces a 20-second pause ($\pm 25\%$ jitter) between chunks to avoid rate limiting and IP blocks from upstream sources.
- **Smart Skip & Idempotent Resumption**: Automatically checks `WHERE game_id NOT IN (SELECT DISTINCT game_id FROM line_scores WHERE deleted_at IS NULL)` so completed games/chunks are instantly skipped.
- **Graceful Interrupt Handling (`Ctrl+C`)**: Captures `SIGINT`/`SIGTERM` via `context.Context`. If interrupted, in-flight scraped games in the active chunk are flushed to the database before cleanly exiting without data loss.
+#### 1. Auto-Detect Missing Box Scores Engine (`importMissingBoxScores`)
+- **Dynamic Database Discovery**: Automatically queries PostgreSQL for any games in the `games` table that lack corresponding records in `line_scores` (`WHERE game_id NOT IN (SELECT DISTINCT game_id FROM line_scores WHERE deleted_at IS NULL)`).
+- **Zero Hardcoding**: Eliminates the need to manually configure date ranges or game IDs when fixing missing data across multiple historical seasons (e.g., 2008, 2013, 2017).
+- **20-Game Batches**: Groups detected missing games into safe 20-game chunks with 2 concurrent workers and staggered worker starts.
+- **Immediate Incremental Persistence**: Scraped data (`line_scores`, `player_game_basic_stats`, `player_game_adv_stats`, `team_game_basic_stats`, `team_game_adv_stats`) is immediately committed and upserted into PostgreSQL after every batch.
+- **Inter-Batch Cool-Off**: Enforces a 20-second pause ($\pm 25\%$ jitter) between batches to maintain compliant request rates against upstream sources.
+- **Defensive URL Construction**: Automatically constructs `/boxscores/{gameID}.html` if a game record has an empty `box_score_url`.
+
+#### 2. Date-Range Chunked Ingestion (`importBoxScores`)
+- **20-Day Temporal Session Chunks**: Large date spans (such as an entire 9-month season) are partitioned into 20-day sliding windows (~80–120 games per chunk).
+- **Off-Season Smart Skipping**: Summer months (July–October) with 0 games are automatically identified and skipped in milliseconds without triggering scraping pauses.
+
+#### 3. Fault Tolerance & Safety Guarantees
+- **Graceful Interrupt Handling (`Ctrl+C`)**: Captures `SIGINT` and `SIGTERM` via `context.Context`. If interrupted, all data from completed batches is safely preserved in PostgreSQL.
+- **Idempotent Resumption**: Re-running `go run . import-data` automatically discovers only the remaining pending games, skipping all previously completed games.
+
+---
+
+### Database Verification Queries
+
+Run the following SQL queries in PostgreSQL to verify data completeness and monitor ingestion progress:
+
+```sql
+-- 1. Check count of remaining missing games (returns 0 when fully complete)
+SELECT count(*) AS remaining_missing_games
+FROM games g
+LEFT JOIN line_scores ls ON g.game_id = ls.game_id AND ls.deleted_at IS NULL
+WHERE ls.game_id IS NULL AND g.deleted_at IS NULL;
+
+-- 2. Inspect recently imported line scores
+SELECT ls.game_id, g.date, ls.team, ls.q1, ls.q2, ls.q3, ls.q4, ls.ot1, ls.total
+FROM line_scores ls
+JOIN games g ON ls.game_id = g.game_id
+ORDER BY ls.updated_at DESC
+LIMIT 20;
+
+-- 3. Verify total games vs total line scores
+SELECT 
+    (SELECT count(*) FROM games WHERE deleted_at IS NULL) AS total_games,
+    (SELECT count(DISTINCT game_id) FROM line_scores WHERE deleted_at IS NULL) AS games_with_boxscores,
+    (SELECT count(*) FROM line_scores WHERE deleted_at IS NULL) AS total_line_scores;
+```
+
+---

 ### Stopping

@@ -273,3 +312,20 @@ go run loadtest.go -n 100 -c 10 -url "http://localhost:8080/api/playeradvancedst
 | Containerization | Docker + Docker Compose |

 ## Workflows
+
+### 1. Ingesting a New Season from Scratch
+1. Set the target season in [`import.go`](file:///Volumes/ROG_BLACK/code/update/NBA_Go/import.go) for `importPlayerTotalsScrape`, `importPlayerAdvanced`, `importGameSchedules`, and `importMarkPlayoffGames`.
+2. Enable schedule and season totals imports in [`main.go`](file:///Volumes/ROG_BLACK/code/update/NBA_Go/main.go).
+3. Run `go run . import-data` to ingest schedules and season totals.
+4. Run `importMissingBoxScores` to automatically ingest all game box scores and line scores in 20-game chunks.
+
+### 2. Auto-Detecting & Filling Data Gaps
+1. Ensure `importMissingBoxScores(db)` is active in [`main.go`](file:///Volumes/ROG_BLACK/code/update/NBA_Go/main.go).
+2. Run `go run . import-data`.
+3. The engine automatically finds any missing games in PostgreSQL across all seasons, splits them into 20-game batches, and ingests them with immediate DB commits.
+
+### 3. Local Development & API Testing
+1. Start the local stack with `make up` or `docker-compose -f docker-compose.local.yml up -d`.
+2. Open Swagger documentation at `http://localhost:8081/swagger/index.html`.
+3. View Grafana metrics dashboards at `http://localhost:3001` (login: `admin` / `testing`).
+
@@ -16,7 +16,7 @@ import (

 // importPlayerAdvanced fetches and stores advanced stats for seasons
 func importPlayerAdvanced(db *gorm.DB) {
-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2009; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
@@ -28,7 +28,7 @@ func importPlayerAdvanced(db *gorm.DB) {

 // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons
 func importPlayerAdvancedPlayoffs(db *gorm.DB) {
-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2009; season++ {
 		if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
 			log.Printf("advanced import failed for %d: %v", season, err)
 		}
@@ -40,7 +40,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) {

 // importPlayerTotalsScrape fetches & stores scraped regular-season total stats
 func importPlayerTotalsScrape(db *gorm.DB) {
-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2009; season++ {
 		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
 			log.Printf("scraped totals import failed for %d: %v", season, err)
 		}
@@ -52,7 +52,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {

 // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
 func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2009; season++ {
 		if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
 			log.Printf("scraped playoffs import failed for %d: %v", season, err)
 		}
@@ -66,7 +66,7 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
 func importGameSchedules(db *gorm.DB) {
 	months := []string{"october", "november", "december", "january", "february", "march", "april", "may", "june"}

-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2009; season++ {
 		log.Printf("--- Starting Game Schedule Import for Season: %d ---", season)
 		for _, month := range months {
 			if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil {
@@ -86,8 +86,8 @@ func importBoxScores(db *gorm.DB) {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()

-	from := time.Date(2000, time.October, 31, 0, 0, 0, 0, time.UTC)
-	to := time.Date(2001, time.June, 30, 23, 59, 59, 0, time.UTC)
+	from := time.Date(2008, time.February, 28, 0, 0, 0, 0, time.UTC)
+	to := time.Date(2008, time.December, 31, 23, 59, 59, 0, time.UTC)

 	chunkDays := 20             // 20-day session chunks
 	coolOff := 20 * time.Second // 20s cool-off with jitter between chunks
@@ -104,6 +104,26 @@ func importBoxScores(db *gorm.DB) {
 	log.Println("--- Finished Box Score Data Import ---")
 }

+// importMissingBoxScores auto-detects games in the database lacking line scores and scrapes all missing box scores
+// in safe 20-game chunks with worker rate limiting, cool-off pauses, and immediate database upserts.
+func importMissingBoxScores(db *gorm.DB) {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	batchSize := 20             // 20 games per batch chunk
+	coolOff := 20 * time.Second // 20s cool-off with jitter between batches
+
+	if err := services.FetchAndStoreMissingBoxScores(ctx, db, batchSize, coolOff); err != nil {
+		if errors.Is(err, context.Canceled) {
+			log.Println("👋 Missing box score import interrupted by user. Safe to resume anytime!")
+			return
+		}
+		log.Fatalf("Missing box score import failed: %v", err)
+	}
+
+	log.Println("--- Finished Missing Box Score Import ---")
+}
+
 // importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons.
 func importPlayerShotCharts(db *gorm.DB) {
 	log.Println("--- Starting Player Shot Chart Import from Predefined List ---")
@@ -184,7 +204,7 @@ func importPlayerShotCharts(db *gorm.DB) {

 // importMarkPlayoffGames marks games as playoff using the dedicated Basketball Reference playoff schedule.
 func importMarkPlayoffGames(db *gorm.DB) {
-	for season := 2001; season <= 2001; season++ {
+	for season := 2008; season <= 2008; season++ {
 		if err := services.FetchAndMarkPlayoffGames(db, season); err != nil {
 			log.Printf("playoff marking failed for %d: %v", season, err)
 		}
@@ -65,8 +65,8 @@ func main() {
 		// importGameSchedules(db)
 		// log.Println("🎉 Game Imports completed successfully 🏀")

-		importBoxScores(db)
-		log.Println("🎉 Related Box Score Imports completed successfully 📦")
+		importMissingBoxScores(db)
+		log.Println("🎉 Missing Box Score Imports completed successfully 📦")

 		importMarkPlayoffGames(db)
 		log.Println("🎉 Playoff games marked successfully 🏆")
@@ -146,6 +146,95 @@ func FetchAndStoreBoxScoreDataChunked(
 	return nil
 }

+// FetchAndStoreMissingBoxScores auto-detects all games in the database lacking line_scores,
+// groups them into batches, and scrapes their box scores with rate limiting and immediate upserts.
+func FetchAndStoreMissingBoxScores(
+	ctx context.Context,
+	db *gorm.DB,
+	batchSize int,
+	coolOffBase time.Duration,
+) error {
+	if batchSize <= 0 {
+		batchSize = 20
+	}
+
+	var missingGames []models.Game
+	err := db.Where("game_id NOT IN (SELECT DISTINCT game_id FROM line_scores WHERE deleted_at IS NULL)").
+		Where("deleted_at IS NULL").
+		Order("date ASC").
+		Find(&missingGames).Error
+
+	if err != nil {
+		return fmt.Errorf("failed to query missing games: %w", err)
+	}
+
+	totalMissing := len(missingGames)
+	if totalMissing == 0 {
+		log.Println("🎉 All games in the database already have box score and line score data! Nothing to scrape.")
+		return nil
+	}
+
+	totalBatches := (totalMissing + batchSize - 1) / batchSize
+	log.Printf("🔍 Auto-Detect: Found %d missing games across all seasons. Processing in %d batches (%d games/batch).",
+		totalMissing, totalBatches, batchSize)
+
+	totalSaved := 0
+
+	for i := 0; i < totalBatches; i++ {
+		select {
+		case <-ctx.Done():
+			log.Println("🛑 Interrupt received. Halting missing box score scraping.")
+			return ctx.Err()
+		default:
+		}
+
+		startIdx := i * batchSize
+		endIdx := startIdx + batchSize
+		if endIdx > totalMissing {
+			endIdx = totalMissing
+		}
+
+		batchGames := missingGames[startIdx:endIdx]
+		batchNumber := i + 1
+
+		log.Printf("\n📦 ========================================================")
+		log.Printf("📦 [Batch %d/%d] Processing %d games (%s to %s)",
+			batchNumber, totalBatches, len(batchGames),
+			batchGames[0].Date.Format("2006-01-02"),
+			batchGames[len(batchGames)-1].Date.Format("2006-01-02"))
+		log.Printf("📦 ========================================================")
+
+		// Scrape batch with worker pool
+		results := processGamesWithWorkers(ctx, batchGames)
+
+		// Immediate database upsert
+		if len(results) > 0 {
+			log.Printf("💾 Saving and upserting data for %d games from Batch %d into PostgreSQL...", len(results), batchNumber)
+			if err := persistScrapedResults(db, results); err != nil {
+				log.Printf("❌ Failed to upsert results for batch %d: %v", batchNumber, err)
+				return err
+			}
+			totalSaved += len(results)
+			log.Printf("✅ [Batch %d/%d] Successfully saved %d games. (Total progress: %d/%d)",
+				batchNumber, totalBatches, len(results), totalSaved, totalMissing)
+		}
+
+		if ctx.Err() != nil {
+			log.Printf("🛑 Process interrupted! All data scraped up to Batch %d was safely committed to DB.", batchNumber)
+			return ctx.Err()
+		}
+
+		// Cool-off pause between batches
+		if i < totalBatches-1 {
+			log.Printf("😴 Cool-off period: Pausing before Batch %d/%d...", batchNumber+1, totalBatches)
+			utils.SleepWithJitter(coolOffBase)
+		}
+	}
+
+	log.Printf("\n🎉 All Missing Box Scores Finished! Successfully processed %d total games.", totalSaved)
+	return nil
+}
+
 // processGamesWithWorkers runs the worker pool for a slice of games.
 func processGamesWithWorkers(ctx context.Context, games []models.Game) []ScrapedResult {
 	jobs := make(chan models.Game, len(games))
@@ -243,8 +332,11 @@ func scrapeAndParseWorker(ctx context.Context, id int, jobs <-chan models.Game,
 				return
 			}

-			log.Printf("🐝  Worker %d: Processing game %s", id, game.GameID)
-			fullURL := boxScoreURLBase + game.BoxScoreURL
+			boxScoreURL := game.BoxScoreURL
+			if boxScoreURL == "" {
+				boxScoreURL = fmt.Sprintf("/boxscores/%s.html", game.GameID)
+			}
+			fullURL := boxScoreURLBase + boxScoreURL

 			utils.SleepWithJitter(baseDelay)
 			time.Sleep(2500 * time.Millisecond)