diff --git a/.DS_Store b/.DS_Store index 31742f4..2c6610e 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/import-state.json b/.github/import-state.json new file mode 100644 index 0000000..00c897e --- /dev/null +++ b/.github/import-state.json @@ -0,0 +1,8 @@ +{ + "current_season": 2006, + "current_from": "2006-10-01", + "current_to": "2006-12-15", + "target_end": "2026-06-15", + "window_days": 60, + "completed": false +} diff --git a/.github/scripts/generate-import.sh b/.github/scripts/generate-import.sh new file mode 100755 index 0000000..cccb9c2 --- /dev/null +++ b/.github/scripts/generate-import.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Generates import.go with the given season/date parameters. +# Usage: generate-import.sh SEASON GO_MONTHS FROM_YEAR FROM_MONTH FROM_DAY TO_YEAR TO_MONTH TO_DAY +set -e + +SEASON="$1" +GO_MONTHS="$2" +FROM_YEAR="$3" +FROM_MONTH="$4" +FROM_DAY="$5" +TO_YEAR="$6" +TO_MONTH="$7" +TO_DAY="$8" + +cat > import.go << EOF +package main + +import ( + "log" + "time" + "fmt" + + "github.com/nprasad2077/NBA_Go/services" + "github.com/nprasad2077/NBA_Go/utils" + "gorm.io/gorm" +) + +// importPlayerAdvanced fetches and stores advanced stats for seasons +func importPlayerAdvanced(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1000 * time.Millisecond) + } +} + +// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons +func importPlayerAdvancedPlayoffs(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced Playoffs import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1500 * time.Millisecond) + } +} + +// importPlayerTotalsScrape fetches & stores scraped regular-season total stats +func importPlayerTotalsScrape(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { + log.Printf("scraped totals import failed for %d: %v", season, err) + } + log.Printf("Player Totals import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1250 * time.Millisecond) + } +} + +// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats +func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { + log.Printf("scraped playoffs import failed for %d: %v", season, err) + } + log.Printf("Player Playoffs Totals import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1700 * time.Millisecond) + } +} + +// importGameSchedules fetches and stores game schedules +func importGameSchedules(db *gorm.DB) { + months := []string{${GO_MONTHS}} + + for season := ${SEASON}; season <= ${SEASON}; season++ { + log.Printf("--- Starting Game Schedule Import for Season: %d ---", season) + for _, month := range months { + if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil { + log.Printf("Game schedule import failed for %s %d: %v", month, season, err) + } + log.Printf("Game schedule import for %s, %d complete.", month, season) + time.Sleep(2500 * time.Millisecond) + utils.SleepWithJitter(1800 * time.Millisecond) + } + log.Printf("--- Finished Game Schedule Import for Season: %d ---", season) + } +} + +// importBoxScores fetches and stores all box score data (line scores, player/team stats) +// for games within a recent date range. +func importBoxScores(db *gorm.DB) { + from := time.Date(${FROM_YEAR}, time.${FROM_MONTH}, ${FROM_DAY}, 0, 0, 0, 0, time.UTC) + to := time.Date(${TO_YEAR}, time.${TO_MONTH}, ${TO_DAY}, 5, 30, 0, 0, time.UTC) + + dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", + from.Format("January 2, 2006"), + to.Format("January 2, 2006")) + + log.Println(dateRangeComment) + + if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { + log.Fatalf("Box score import failed: %v", err) + } + + log.Println("--- Finished Box Score Data Import ---") +} + +// importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons. +func importPlayerShotCharts(db *gorm.DB) { + log.Println("--- Starting Player Shot Chart Import from Predefined List ---") + + startSeason := 2026 + endSeason := 2017 + + targetPlayerIDs := []string{ + "jamesle01", + "curryst01", + "jokicni01", + "doncilu01", + "hardeja01", + "irvinky01", + "duranke01", + "youngtr01", + "gilgesh01", + "brunsja01", + "edwaran01", + "mitchdo01", + "derozde01", + "westbru01", + "butleji01", + "davisan02", + "leonaka01", + "tatumja01", + } + + if len(targetPlayerIDs) == 0 { + log.Println("⚠️ Player ID list is empty. Skipping shot chart import.") + return + } + + log.Printf("Found %d players in the predefined list. Starting shot chart import for seasons %d down to %d.", len(targetPlayerIDs), startSeason, endSeason) + + for i, playerID := range targetPlayerIDs { + log.Printf("(%d/%d) Importing shot charts for player: %s", i+1, len(targetPlayerIDs), playerID) + + err := services.FetchAndStoreShotChartScrapedForPlayer(db, playerID, startSeason, endSeason) + if err != nil { + log.Printf("❌ Shot chart import failed for player %s: %v", playerID, err) + } + + utils.SleepWithJitter(10000 * time.Millisecond) + } + + log.Println("--- Finished Player Shot Chart Import from Predefined List ---") +} +EOF diff --git a/.github/workflows/advance-import.yml b/.github/workflows/advance-import.yml new file mode 100644 index 0000000..82a4b8d --- /dev/null +++ b/.github/workflows/advance-import.yml @@ -0,0 +1,125 @@ +name: Advance NBA Import Window + +on: + schedule: + # Runs every 40 minutes. Adjust to slow down: + # "0 * * * *" = hourly | "0 */2 * * *" = every 2h | "0 */3 * * *" = every 3h + - cron: "*/40 * * * *" + workflow_dispatch: # Manual trigger + +jobs: + advance-import: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/remote' + steps: + - name: Checkout remote branch + uses: actions/checkout@v4 + with: + ref: remote + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Advance import window + id: advance + run: | + set -e + STATE_FILE=".github/import-state.json" + + COMPLETED=$(jq -r '.completed' "$STATE_FILE") + if [ "$COMPLETED" = "true" ]; then + echo "All imports completed. Nothing to do." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" + + # Read state + SEASON=$(jq -r '.current_season' "$STATE_FILE") + FROM=$(jq -r '.current_from' "$STATE_FILE") + TO=$(jq -r '.current_to' "$STATE_FILE") + TARGET_END=$(jq -r '.target_end' "$STATE_FILE") + WINDOW_DAYS=$(jq -r '.window_days' "$STATE_FILE") + + echo "Generating import.go for Season $SEASON: $FROM → $TO" + + # --- Build months list from date range --- + MONTHS=() + cursor="$FROM" + while [[ "$cursor" < "$TO" || "$cursor" == "$TO" ]]; do + m=$(date -d "$cursor" +%B | tr '[:upper:]' '[:lower:]') + # Add if not already present + if [[ ! " ${MONTHS[*]} " =~ " $m " ]]; then + MONTHS+=("$m") + fi + cursor=$(date -d "$cursor + 32 days" +%Y-%m-01) + done + # Ensure TO's month is included + to_m=$(date -d "$TO" +%B | tr '[:upper:]' '[:lower:]') + if [[ ! " ${MONTHS[*]} " =~ " $to_m " ]]; then + MONTHS+=("$to_m") + fi + + # Format as Go string slice entries + GO_MONTHS="" + for m in "${MONTHS[@]}"; do + GO_MONTHS="${GO_MONTHS}\"${m}\", " + done + GO_MONTHS="${GO_MONTHS%, }" + + # Parse date components + FROM_YEAR=$(date -d "$FROM" +%Y) + FROM_MONTH=$(date -d "$FROM" +%B) + FROM_DAY=$(date -d "$FROM" +%-d) + TO_YEAR=$(date -d "$TO" +%Y) + TO_MONTH=$(date -d "$TO" +%B) + TO_DAY=$(date -d "$TO" +%-d) + + # --- Generate import.go --- + bash .github/scripts/generate-import.sh \ + "$SEASON" "$GO_MONTHS" \ + "$FROM_YEAR" "$FROM_MONTH" "$FROM_DAY" \ + "$TO_YEAR" "$TO_MONTH" "$TO_DAY" + + # --- Calculate next window --- + NEXT_FROM="$TO" + NEXT_TO=$(date -d "$NEXT_FROM + ${WINDOW_DAYS} days" +%Y-%m-%d) + NEXT_SEASON=$SEASON + + # Season Y runs Oct Y to ~June Y+1. If past June of Y+1, advance season. + SEASON_END_DATE="$((SEASON + 1))-06-30" + if [[ "$NEXT_FROM" > "$SEASON_END_DATE" ]]; then + NEXT_SEASON=$((SEASON + 1)) + NEXT_FROM="${NEXT_SEASON}-10-01" + NEXT_TO=$(date -d "$NEXT_FROM + ${WINDOW_DAYS} days" +%Y-%m-%d) + fi + + # Cap at season boundary + NEXT_SEASON_END="$((NEXT_SEASON + 1))-06-30" + if [[ "$NEXT_TO" > "$NEXT_SEASON_END" ]]; then + NEXT_TO="$NEXT_SEASON_END" + fi + + # Check completion + NEXT_COMPLETED="false" + if [[ "$NEXT_FROM" > "$TARGET_END" || "$NEXT_FROM" == "$TARGET_END" ]]; then + NEXT_COMPLETED="true" + fi + + # Update state + jq \ + --argjson season "$NEXT_SEASON" \ + --arg from "$NEXT_FROM" \ + --arg to "$NEXT_TO" \ + --argjson completed "$([ "$NEXT_COMPLETED" = "true" ] && echo true || echo false)" \ + '.current_season = $season | .current_from = $from | .current_to = $to | .completed = $completed' \ + "$STATE_FILE" > tmp.json && mv tmp.json "$STATE_FILE" + + echo "Next: Season $NEXT_SEASON, $NEXT_FROM → $NEXT_TO (done: $NEXT_COMPLETED)" + + - name: Commit and push + if: steps.advance.outputs.skip != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add import.go .github/import-state.json + git commit -m "chore: import window $(jq -r '.current_from' .github/import-state.json) [season $(jq -r '.current_season' .github/import-state.json)]" + git push origin remote diff --git a/.gitignore b/.gitignore index 111ec7b..64b41f9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ docs/notes docs/postman docs/kiro.md -docs/digital_ocean/digitaal ocean llms.txt \ No newline at end of file +docs/digital_ocean/digitaal ocean llms.txt + +.docs \ No newline at end of file diff --git a/import.go b/import.go index bd1d0e2..a96feaa 100644 --- a/import.go +++ b/import.go @@ -138,7 +138,7 @@ func importPlayerShotCharts(db *gorm.DB) { "gilgesh01", "brunsja01", "edwaran01", - // "mitchdo01", + "mitchdo01", // "bookede01", "derozde01", // "aldrila01", diff --git a/main.go b/main.go index c17c65a..faa472b 100644 --- a/main.go +++ b/main.go @@ -50,17 +50,17 @@ func main() { // Run all DB migrations + import steps exactly once db := config.InitDB(true) - importPlayerAdvanced(db) - log.Println("🎉 Player Advanced Import completed successfully") + // importPlayerAdvanced(db) + // log.Println("🎉 Player Advanced Import completed successfully") - importPlayerAdvancedPlayoffs(db) - log.Println("🎉 Player Advanced Playoffs Import completed successfully") + // importPlayerAdvancedPlayoffs(db) + // log.Println("🎉 Player Advanced Playoffs Import completed successfully") - importPlayerTotalsScrape(db) - log.Println("🎉 Player Totals (scraped) Import completed successfully") + // importPlayerTotalsScrape(db) + // log.Println("🎉 Player Totals (scraped) Import completed successfully") - importPlayerTotalsPlayoffsScrape(db) - log.Println("🎉 Player Playoffs (scraped) Import completed successfully") + // importPlayerTotalsPlayoffsScrape(db) + // log.Println("🎉 Player Playoffs (scraped) Import completed successfully") importGameSchedules(db) log.Println("🎉 Game Imports completed successfully 🏀") @@ -68,8 +68,8 @@ func main() { importBoxScores(db) log.Println("🎉 Related Box Score Imports completed successfully 📦") - importPlayerShotCharts(db) - log.Println("🎉 Player Shot Chart Import completed successfully 🎯") + // importPlayerShotCharts(db) + // log.Println("🎉 Player Shot Chart Import completed successfully 🎯") log.Println("🏀 ALL Imports completed successfully ✅ 🙌") return