From 8500ed6c8bf0ed29f399f04130924cef508196de Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Thu, 11 Jun 2026 02:34:36 -0500 Subject: [PATCH] workflows update --- .DS_Store | Bin 30724 -> 32772 bytes .github/import-state.json | 8 ++ .github/scripts/generate-import.sh | 161 +++++++++++++++++++++++++++ .github/workflows/advance-import.yml | 125 +++++++++++++++++++++ .gitignore | 4 +- import.go | 2 +- main.go | 20 ++-- 7 files changed, 308 insertions(+), 12 deletions(-) create mode 100644 .github/import-state.json create mode 100755 .github/scripts/generate-import.sh create mode 100644 .github/workflows/advance-import.yml diff --git a/.DS_Store b/.DS_Store index 31742f48e16b66eb937418ad6d008a3d1ef3a70c..2c6610ea24f3e97793301979676f3f4eda869fb5 100644 GIT binary patch delta 759 zcmZqqz}V8jB*VbK!01$(V#vUd0c3IjF^Cr6VPGiWnJ8l{S_tH^0|giu@)*(>G8xiy zipvT%7EWWI*ucA)or8sgk$>`;beYY$nQ_dUc?4Q{>ZO4?SQ+#fQW)|Xk{OC~(hY-? z^K%P;q6`d7GulvPbMswXl5+BsfTA4T&1b@W-yU~FSCK+MML`C#4NM0ZCi5sr)gMs= zn!^HgM>^2eCCK)yQn6+5fY^g1kFW)sJc=E*4nU)Up#n7<7-FRiNhtngP_GxkRD$dX zDFKEZJ`hKs8I+rka0+f!C=Ox>fH|m)p%@tG`3xzWrzphnZ0?ZaWSY!vAUOGszW8L} z43*9N2Co>IzDsQWZOF(*tC%Syy`+{ zN9k3d!JH7oGq{1YD=f8bmML&yVPOO&gvl8t6DKcBvYPB*zhH7=hU8{OT^S}|iez8{ zVpbq#+5A!G9up($Cff&0PzC3h7}>$LZ;2P3{3q?gG8xiy z^HW?l7EWWI*ucA)or8sgk#}>Oa4XNohAL)8(aAjuQnB-=0rjvj=rN=N)tBVvySOCf zB@&al z4For*7~Ey#`8Ek`ZVE#_L-OW_o?*~1-lEZNNDnfT#3y!0rmWoFUJc`K3yR?`D#4>WQ7DC4P@`^ z6M75PhG7EdX6D5AY?JR-T%61!AT!w^LIxP33`{`G3dAgv4Gh#7StlDgXih$#D>*qJ zK%9|%vQEM+Mvl!I0&&cn`O8-_PR>qmM>aqLsF4$hfuMjJNVvkHX|qg*3kwI(Y*6e~ QOisy}xcNgW6BDvt011V!Z diff --git a/.github/import-state.json b/.github/import-state.json new file mode 100644 index 0000000..00c897e --- /dev/null +++ b/.github/import-state.json @@ -0,0 +1,8 @@ +{ + "current_season": 2006, + "current_from": "2006-10-01", + "current_to": "2006-12-15", + "target_end": "2026-06-15", + "window_days": 60, + "completed": false +} diff --git a/.github/scripts/generate-import.sh b/.github/scripts/generate-import.sh new file mode 100755 index 0000000..cccb9c2 --- /dev/null +++ b/.github/scripts/generate-import.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Generates import.go with the given season/date parameters. +# Usage: generate-import.sh SEASON GO_MONTHS FROM_YEAR FROM_MONTH FROM_DAY TO_YEAR TO_MONTH TO_DAY +set -e + +SEASON="$1" +GO_MONTHS="$2" +FROM_YEAR="$3" +FROM_MONTH="$4" +FROM_DAY="$5" +TO_YEAR="$6" +TO_MONTH="$7" +TO_DAY="$8" + +cat > import.go << EOF +package main + +import ( + "log" + "time" + "fmt" + + "github.com/nprasad2077/NBA_Go/services" + "github.com/nprasad2077/NBA_Go/utils" + "gorm.io/gorm" +) + +// importPlayerAdvanced fetches and stores advanced stats for seasons +func importPlayerAdvanced(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1000 * time.Millisecond) + } +} + +// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons +func importPlayerAdvancedPlayoffs(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { + log.Printf("advanced import failed for %d: %v", season, err) + } + log.Printf("Advanced Playoffs import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1500 * time.Millisecond) + } +} + +// importPlayerTotalsScrape fetches & stores scraped regular-season total stats +func importPlayerTotalsScrape(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { + log.Printf("scraped totals import failed for %d: %v", season, err) + } + log.Printf("Player Totals import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1250 * time.Millisecond) + } +} + +// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats +func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { + for season := 2026; season <= 2026; season++ { + if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { + log.Printf("scraped playoffs import failed for %d: %v", season, err) + } + log.Printf("Player Playoffs Totals import for season: %d", season) + time.Sleep(2000 * time.Millisecond) + utils.SleepWithJitter(1700 * time.Millisecond) + } +} + +// importGameSchedules fetches and stores game schedules +func importGameSchedules(db *gorm.DB) { + months := []string{${GO_MONTHS}} + + for season := ${SEASON}; season <= ${SEASON}; season++ { + log.Printf("--- Starting Game Schedule Import for Season: %d ---", season) + for _, month := range months { + if err := services.FetchAndStoreGameSchedule(db, season, month); err != nil { + log.Printf("Game schedule import failed for %s %d: %v", month, season, err) + } + log.Printf("Game schedule import for %s, %d complete.", month, season) + time.Sleep(2500 * time.Millisecond) + utils.SleepWithJitter(1800 * time.Millisecond) + } + log.Printf("--- Finished Game Schedule Import for Season: %d ---", season) + } +} + +// importBoxScores fetches and stores all box score data (line scores, player/team stats) +// for games within a recent date range. +func importBoxScores(db *gorm.DB) { + from := time.Date(${FROM_YEAR}, time.${FROM_MONTH}, ${FROM_DAY}, 0, 0, 0, 0, time.UTC) + to := time.Date(${TO_YEAR}, time.${TO_MONTH}, ${TO_DAY}, 5, 30, 0, 0, time.UTC) + + dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", + from.Format("January 2, 2006"), + to.Format("January 2, 2006")) + + log.Println(dateRangeComment) + + if err := services.FetchAndStoreBoxScoreDataForDateRange(db, from, to); err != nil { + log.Fatalf("Box score import failed: %v", err) + } + + log.Println("--- Finished Box Score Data Import ---") +} + +// importPlayerShotCharts fetches shot charts for a PREDEFINED list of players for a given range of seasons. +func importPlayerShotCharts(db *gorm.DB) { + log.Println("--- Starting Player Shot Chart Import from Predefined List ---") + + startSeason := 2026 + endSeason := 2017 + + targetPlayerIDs := []string{ + "jamesle01", + "curryst01", + "jokicni01", + "doncilu01", + "hardeja01", + "irvinky01", + "duranke01", + "youngtr01", + "gilgesh01", + "brunsja01", + "edwaran01", + "mitchdo01", + "derozde01", + "westbru01", + "butleji01", + "davisan02", + "leonaka01", + "tatumja01", + } + + if len(targetPlayerIDs) == 0 { + log.Println("⚠️ Player ID list is empty. Skipping shot chart import.") + return + } + + log.Printf("Found %d players in the predefined list. Starting shot chart import for seasons %d down to %d.", len(targetPlayerIDs), startSeason, endSeason) + + for i, playerID := range targetPlayerIDs { + log.Printf("(%d/%d) Importing shot charts for player: %s", i+1, len(targetPlayerIDs), playerID) + + err := services.FetchAndStoreShotChartScrapedForPlayer(db, playerID, startSeason, endSeason) + if err != nil { + log.Printf("❌ Shot chart import failed for player %s: %v", playerID, err) + } + + utils.SleepWithJitter(10000 * time.Millisecond) + } + + log.Println("--- Finished Player Shot Chart Import from Predefined List ---") +} +EOF diff --git a/.github/workflows/advance-import.yml b/.github/workflows/advance-import.yml new file mode 100644 index 0000000..82a4b8d --- /dev/null +++ b/.github/workflows/advance-import.yml @@ -0,0 +1,125 @@ +name: Advance NBA Import Window + +on: + schedule: + # Runs every 40 minutes. Adjust to slow down: + # "0 * * * *" = hourly | "0 */2 * * *" = every 2h | "0 */3 * * *" = every 3h + - cron: "*/40 * * * *" + workflow_dispatch: # Manual trigger + +jobs: + advance-import: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/remote' + steps: + - name: Checkout remote branch + uses: actions/checkout@v4 + with: + ref: remote + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Advance import window + id: advance + run: | + set -e + STATE_FILE=".github/import-state.json" + + COMPLETED=$(jq -r '.completed' "$STATE_FILE") + if [ "$COMPLETED" = "true" ]; then + echo "All imports completed. Nothing to do." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" + + # Read state + SEASON=$(jq -r '.current_season' "$STATE_FILE") + FROM=$(jq -r '.current_from' "$STATE_FILE") + TO=$(jq -r '.current_to' "$STATE_FILE") + TARGET_END=$(jq -r '.target_end' "$STATE_FILE") + WINDOW_DAYS=$(jq -r '.window_days' "$STATE_FILE") + + echo "Generating import.go for Season $SEASON: $FROM → $TO" + + # --- Build months list from date range --- + MONTHS=() + cursor="$FROM" + while [[ "$cursor" < "$TO" || "$cursor" == "$TO" ]]; do + m=$(date -d "$cursor" +%B | tr '[:upper:]' '[:lower:]') + # Add if not already present + if [[ ! " ${MONTHS[*]} " =~ " $m " ]]; then + MONTHS+=("$m") + fi + cursor=$(date -d "$cursor + 32 days" +%Y-%m-01) + done + # Ensure TO's month is included + to_m=$(date -d "$TO" +%B | tr '[:upper:]' '[:lower:]') + if [[ ! " ${MONTHS[*]} " =~ " $to_m " ]]; then + MONTHS+=("$to_m") + fi + + # Format as Go string slice entries + GO_MONTHS="" + for m in "${MONTHS[@]}"; do + GO_MONTHS="${GO_MONTHS}\"${m}\", " + done + GO_MONTHS="${GO_MONTHS%, }" + + # Parse date components + FROM_YEAR=$(date -d "$FROM" +%Y) + FROM_MONTH=$(date -d "$FROM" +%B) + FROM_DAY=$(date -d "$FROM" +%-d) + TO_YEAR=$(date -d "$TO" +%Y) + TO_MONTH=$(date -d "$TO" +%B) + TO_DAY=$(date -d "$TO" +%-d) + + # --- Generate import.go --- + bash .github/scripts/generate-import.sh \ + "$SEASON" "$GO_MONTHS" \ + "$FROM_YEAR" "$FROM_MONTH" "$FROM_DAY" \ + "$TO_YEAR" "$TO_MONTH" "$TO_DAY" + + # --- Calculate next window --- + NEXT_FROM="$TO" + NEXT_TO=$(date -d "$NEXT_FROM + ${WINDOW_DAYS} days" +%Y-%m-%d) + NEXT_SEASON=$SEASON + + # Season Y runs Oct Y to ~June Y+1. If past June of Y+1, advance season. + SEASON_END_DATE="$((SEASON + 1))-06-30" + if [[ "$NEXT_FROM" > "$SEASON_END_DATE" ]]; then + NEXT_SEASON=$((SEASON + 1)) + NEXT_FROM="${NEXT_SEASON}-10-01" + NEXT_TO=$(date -d "$NEXT_FROM + ${WINDOW_DAYS} days" +%Y-%m-%d) + fi + + # Cap at season boundary + NEXT_SEASON_END="$((NEXT_SEASON + 1))-06-30" + if [[ "$NEXT_TO" > "$NEXT_SEASON_END" ]]; then + NEXT_TO="$NEXT_SEASON_END" + fi + + # Check completion + NEXT_COMPLETED="false" + if [[ "$NEXT_FROM" > "$TARGET_END" || "$NEXT_FROM" == "$TARGET_END" ]]; then + NEXT_COMPLETED="true" + fi + + # Update state + jq \ + --argjson season "$NEXT_SEASON" \ + --arg from "$NEXT_FROM" \ + --arg to "$NEXT_TO" \ + --argjson completed "$([ "$NEXT_COMPLETED" = "true" ] && echo true || echo false)" \ + '.current_season = $season | .current_from = $from | .current_to = $to | .completed = $completed' \ + "$STATE_FILE" > tmp.json && mv tmp.json "$STATE_FILE" + + echo "Next: Season $NEXT_SEASON, $NEXT_FROM → $NEXT_TO (done: $NEXT_COMPLETED)" + + - name: Commit and push + if: steps.advance.outputs.skip != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add import.go .github/import-state.json + git commit -m "chore: import window $(jq -r '.current_from' .github/import-state.json) [season $(jq -r '.current_season' .github/import-state.json)]" + git push origin remote diff --git a/.gitignore b/.gitignore index 111ec7b..64b41f9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ docs/notes docs/postman docs/kiro.md -docs/digital_ocean/digitaal ocean llms.txt \ No newline at end of file +docs/digital_ocean/digitaal ocean llms.txt + +.docs \ No newline at end of file diff --git a/import.go b/import.go index bd1d0e2..a96feaa 100644 --- a/import.go +++ b/import.go @@ -138,7 +138,7 @@ func importPlayerShotCharts(db *gorm.DB) { "gilgesh01", "brunsja01", "edwaran01", - // "mitchdo01", + "mitchdo01", // "bookede01", "derozde01", // "aldrila01", diff --git a/main.go b/main.go index c17c65a..faa472b 100644 --- a/main.go +++ b/main.go @@ -50,17 +50,17 @@ func main() { // Run all DB migrations + import steps exactly once db := config.InitDB(true) - importPlayerAdvanced(db) - log.Println("🎉 Player Advanced Import completed successfully") + // importPlayerAdvanced(db) + // log.Println("🎉 Player Advanced Import completed successfully") - importPlayerAdvancedPlayoffs(db) - log.Println("🎉 Player Advanced Playoffs Import completed successfully") + // importPlayerAdvancedPlayoffs(db) + // log.Println("🎉 Player Advanced Playoffs Import completed successfully") - importPlayerTotalsScrape(db) - log.Println("🎉 Player Totals (scraped) Import completed successfully") + // importPlayerTotalsScrape(db) + // log.Println("🎉 Player Totals (scraped) Import completed successfully") - importPlayerTotalsPlayoffsScrape(db) - log.Println("🎉 Player Playoffs (scraped) Import completed successfully") + // importPlayerTotalsPlayoffsScrape(db) + // log.Println("🎉 Player Playoffs (scraped) Import completed successfully") importGameSchedules(db) log.Println("🎉 Game Imports completed successfully 🏀") @@ -68,8 +68,8 @@ func main() { importBoxScores(db) log.Println("🎉 Related Box Score Imports completed successfully 📦") - importPlayerShotCharts(db) - log.Println("🎉 Player Shot Chart Import completed successfully 🎯") + // importPlayerShotCharts(db) + // log.Println("🎉 Player Shot Chart Import completed successfully 🎯") log.Println("🏀 ALL Imports completed successfully ✅ 🙌") return