diff --git a/import.go b/import.go index 1aa3332..26ede63 100644 --- a/import.go +++ b/import.go @@ -11,7 +11,7 @@ import ( // importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 func importPlayerAdvanced(db *gorm.DB) { - for season := 2000; season <= 2014; season++ { + for season := 1991; season <= 1993; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -23,7 +23,7 @@ func importPlayerAdvanced(db *gorm.DB) { // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 func importPlayerAdvancedPlayoffs(db *gorm.DB) { - for season := 2000; season <= 2014; season++ { + for season := 1991; season <= 1993; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -35,7 +35,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) { // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 2000; season <= 2012; season++ { + for season := 1991; season <= 1993; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } @@ -47,7 +47,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 2000; season <= 2012; season++ { + for season := 1991; season <= 1993; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } diff --git a/services/player_shot_chart_scrape_service.go b/services/player_shot_chart_scrape_service.go index e6be630..42c973a 100644 --- a/services/player_shot_chart_scrape_service.go +++ b/services/player_shot_chart_scrape_service.go @@ -17,29 +17,24 @@ import ( "gorm.io/gorm/clause" ) -// FetchAndStoreShotChartScrapedForPlayer scrapes the shot chart pages on -// Basketball‑Reference for one player and upserts every shot into SQLite. -// A *composite* unique key keeps duplicates out: -// -// (player_id, season, date, qtr, time_remaining, top, left) -// -// The model therefore needs a matching unique index (see models package). +// FetchAndStoreShotChartScrapedForPlayer scrapes the shot chart pages on +// Basketball-Reference for one player and batch upserts every shot. func FetchAndStoreShotChartScrapedForPlayer( db *gorm.DB, playerID string, startSeason, endSeason int, ) error { - // loop newest → oldest + // Loop newest → oldest season for season := startSeason; season >= endSeason; season-- { url := fmt.Sprintf( "https://www.basketball-reference.com/players/%s/%s/shooting/%d", playerID[:1], playerID, season, ) - // ─────────────────────── 1) HTTP GET ──────────────────────── + // 1) HTTP GET the page content req, err := http.NewRequest("GET", url, nil) if err != nil { - return fmt.Errorf("request creation error for %d: %w", season, err) + return fmt.Errorf("request creation error for season %d: %w", season, err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ @@ -47,61 +42,66 @@ func FetchAndStoreShotChartScrapedForPlayer( ) resp, err := (&http.Client{}).Do(req) if err != nil { - return fmt.Errorf("HTTP error for %d: %w", season, err) + return fmt.Errorf("HTTP error for season %d: %w", season, err) } if resp.StatusCode != http.StatusOK { resp.Body.Close() - return fmt.Errorf("unexpected status for %d: %s", season, resp.Status) + // This is not an error, just means the player might not have data for that season. + log.Printf("⚠️ Skipping season %d for player %s (Status: %s)", season, playerID, resp.Status) + continue } bodyBytes, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { - return fmt.Errorf("read error for %d: %w", season, err) + return fmt.Errorf("read error for season %d: %w", season, err) } - // ─────────────────── 2) player name (nice to have) ─────────── + // 2) Parse the player name (nice to have) fullDoc, err := goquery.NewDocumentFromReader(bytes.NewReader(bodyBytes)) if err != nil { - return fmt.Errorf("name‑parse error for %d: %w", season, err) + return fmt.Errorf("name-parse error for season %d: %w", season, err) } playerName := fullDoc.Find("#meta span[itemprop='name']").First().Text() if playerName == "" { playerName = playerID } - // ───────────────── 3) commented‑out shot chart HTML ────────── + // 3) Extract the shot chart HTML, which is hidden inside a comment shotHTML := extractCommentedShotChart(bodyBytes) if shotHTML == "" { - log.Printf("⚠️ no shot‑chart comment found for %d", season) + log.Printf("⚠️ No shot-chart comment found for player %s in season %d", playerID, season) continue } doc, err := goquery.NewDocumentFromReader(strings.NewReader(shotHTML)) if err != nil { - return fmt.Errorf("snippet‑parse error for %d: %w", season, err) + return fmt.Errorf("snippet-parse error for season %d: %w", season, err) } wrapper := doc.Find("div#div_shot-chart div#shot-wrapper") if wrapper.Length() == 0 { - log.Printf("⚠️ no shot‑wrapper for %d", season) + log.Printf("⚠️ No shot-wrapper found for player %s in season %d", playerID, season) continue } - // ───────────────────── 4) scrape every tooltip ─────────────── - var firstErr error // surface the first DB failure after the loop + // --- BATCHING LOGIC START --- + // Create a slice to hold all the shot data for the current season. + var shotsToUpsert []models.PlayerShotChart + + // 4) Scrape every tooltip and collect the data into the slice. wrapper.Find("div.tooltip.make, div.tooltip.miss").Each(func(_ int, s *goquery.Selection) { - // position on the court + // Position on the court style, _ := s.Attr("style") parts := strings.Split(style, ";") top := parsePx(parts[0]) left := parsePx(parts[1]) - // tooltip text + // Tooltip text tip, _ := s.Attr("tip") tipParts := strings.Split(tip, "
") - // date, team, opponent - header := tipParts[0] // e.g. "Oct 20, 2021, CHI at DET" - dateSegs := strings.SplitN(header, ", ", 3) // {"Oct 20", "2021", "CHI at DET"} - date := dateSegs[0] + "," + dateSegs[1] // "Oct 20,2021" + // Date, team, opponent + header := tipParts[0] + dateSegs := strings.SplitN(header, ", ", 3) + date := dateSegs[0] + "," + dateSegs[1] var team, opponent string if len(dateSegs) == 3 { @@ -113,18 +113,18 @@ func FetchAndStoreShotChartScrapedForPlayer( } } - // quarter & time remaining + // Quarter & time remaining qt := strings.SplitN(tipParts[1], ",", 2) quarter := qt[0] timeRem := strings.Fields(qt[1])[0] - // result, shot type & distance - rt := strings.Fields(tipParts[2]) // ["Made","2-pointer","..."|"Missed",...] + // Result, shot type & distance + rt := strings.Fields(tipParts[2]) made := rt[0] == "Made" shotType := rt[1] distance := mustAtoi(rt[len(rt)-2]) - // score & lead flag + // Score & lead flag last := strings.Fields(tipParts[3]) sc := strings.Split(last[len(last)-1], "-") teamScore, oppScore := mustAtoi(sc[0]), mustAtoi(sc[1]) @@ -148,31 +148,34 @@ func FetchAndStoreShotChartScrapedForPlayer( Team: team, Season: season, } + // Add the parsed shot object to our slice. + shotsToUpsert = append(shotsToUpsert, shot) + }) + + // 5) Perform the batch upsert operation for the current season. + if len(shotsToUpsert) > 0 { + log.Printf("Attempting to batch upsert %d shots for player %s in season %d...", len(shotsToUpsert), playerID, season) - // ─────────────── 5) upsert / dedup ──────────────── if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{ // MUST match the unique index order - {Name: "player_id"}, - {Name: "season"}, - {Name: "date"}, - {Name: "qtr"}, - {Name: "time_remaining"}, - {Name: "top"}, - {Name: "left"}, + Columns: []clause.Column{ // MUST match the unique index order in the model + {Name: "player_id"}, {Name: "season"}, {Name: "date"}, + {Name: "qtr"}, {Name: "time_remaining"}, {Name: "top"}, {Name: "left"}, }, DoUpdates: clause.AssignmentColumns([]string{ "player_name", "result", "shot_type", "distance_ft", "lead", "team_score", "opponent_team_score", "opponent", "team", }), - }).Create(&shot).Error; err != nil && firstErr == nil { - firstErr = err + }).Create(&shotsToUpsert).Error; err != nil { + // If the batch operation fails, log the error and return it. + return fmt.Errorf("DB upsert error for player %s in season %d: %w", playerID, season, err) } - }) - if firstErr != nil { - return fmt.Errorf("DB upsert error for season %d: %w", season, firstErr) + log.Printf("✅ Successfully batch upserted %d shots for player %s in season %d.", len(shotsToUpsert), playerID, season) + } else { + log.Printf("No shots found to import for player %s in season %d.", playerID, season) } + // --- BATCHING LOGIC END --- } return nil }