NBA_Go/services/player_advanced_playoffs_scrape_service.go

// File: NBA_Go/services/player_advanced_scrape_service.go

package services

import (
    "bytes"
    "fmt"
    "io"
    "log"
    "net/http"
    "strings"

    "github.com/PuerkitoBio/goquery"
    "github.com/nprasad2077/NBA_Go/models"
    "gorm.io/gorm"
    "gorm.io/gorm/clause"
)

const (
    advancedURLFmt        = "https://www.basketball-reference.com/leagues/NBA_%d_advanced.html"
    advancedPlayoffURLFmt = "https://www.basketball-reference.com/playoffs/NBA_%d_advanced.html"
)

// urlForAdvSeason picks the correct URL based on isPlayoff.
func urlForAdvSeason(season int, isPlayoff bool) string {
    if isPlayoff {
        return fmt.Sprintf(advancedPlayoffURLFmt, season)
    }
    return fmt.Sprintf(advancedURLFmt, season)
}

// FetchAndStorePlayerAdvancedScrapedStats scrapes the advanced table
// (regular or playoffs) and upserts into the PlayerAdvancedStat model.
func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff bool) error {
    url := urlForAdvSeason(season, isPlayoff)
    req, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return err
    }
    req.Header.Set("User-Agent", "Mozilla/5.0 (compatible)")
    resp, err := http.DefaultClient.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    htmlBytes, err := io.ReadAll(resp.Body)
    if err != nil {
        return err
    }

    doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
    if err != nil {
        return err
    }

    // pick the right table selector
    var table *goquery.Selection
    if isPlayoff {
        table = doc.Find("#div_advanced_stats table#advanced_stats")
    } else {
        table = doc.Find("#div_advanced table#advanced")
    }
    if table.Length() == 0 {
        return fmt.Errorf("could not find advanced stats table")
    }

    // 1. collect the data-stat keys in header order
    var headers []string
    table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
        if stat, ok := th.Attr("data-stat"); ok && stat != "" {
            headers = append(headers, stat)
        }
    })
    // add our appended-player column
    headers = append(headers, "player-additional")

    // 2. iterate each row
    table.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) {
        if tr.HasClass("thead") {
            return // skip repeated headers
        }
        cells := tr.Find("th, td")
        data := make(map[string]string, len(headers))
        var playerID string

        cells.Each(func(i int, cell *goquery.Selection) {
            key := headers[i]
            data[key] = strings.TrimSpace(cell.Text())
            if id, ok := cell.Attr("data-append-csv"); ok {
                playerID = id
            }
        })
        if playerID == "" {
            return // not a data row
        }
        data["player-additional"] = playerID

        // 4) pick the correct keys for ExternalID, PlayerName, Team, depending on season vs. playoff
        //    - playoffs tables still use "rk", "player", "team_id"
        //    - regular‐season advanced uses "ranker", "name_display", "team_name_abbr"
        extID := mustAtoi(data["rk"])
        if extID == 0 {
            extID = mustAtoi(data["ranker"])
        }

        playerName := data["player"]
        if playerName == "" {
            playerName = data["name_display"]
        }

        teamID := data["team_id"]
        if teamID == "" {
            teamID = data["team_name_abbr"]
        }

        // 5) map into your GORM model
        stat := models.PlayerAdvancedStat{
            ExternalID:         extID,
            PlayerID:           playerID,
            PlayerName:         playerName,
            Position:           data["pos"],
            Age:                mustAtoi(data["age"]),
            Games:              mustAtoi(data["g"]),
            GamesStarted:       mustAtoi(data["games_started"]),
            MinutesPlayed:      mustAtoi(data["mp"]),
            PER:                mustParseFloat(data["per"]),
            TSPercent:          mustParseFloat(data["ts_pct"]),
            ThreePAR:           mustParseFloat(data["fg3a_per_fga_pct"]),
            FTR:                mustParseFloat(data["fta_per_fga_pct"]),
            OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
            DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
            TotalRBPercent:     mustParseFloat(data["trb_pct"]),
            AssistPercent:      mustParseFloat(data["ast_pct"]),
            StealPercent:       mustParseFloat(data["stl_pct"]),
            BlockPercent:       mustParseFloat(data["blk_pct"]),
            TurnoverPercent:    mustParseFloat(data["tov_pct"]),
            UsagePercent:       mustParseFloat(data["usg_pct"]),
            OffensiveWS:        mustParseFloat(data["ows"]),
            DefensiveWS:        mustParseFloat(data["dws"]),
            WinShares:          mustParseFloat(data["ws"]),
            WinSharesPer:       mustParseFloat(data["ws_per_48"]),
            OffensiveBox:       mustParseFloat(data["obpm"]),
            DefensiveBox:       mustParseFloat(data["dbpm"]),
            Box:                mustParseFloat(data["bpm"]),
            VORP:               mustParseFloat(data["vorp"]),
            Team:               teamID,
            Season:             season,
            IsPlayoff:          isPlayoff,
        }

        // 6) upsert on (player_id, season, team, is_playoff)
        if err := db.Clauses(clause.OnConflict{
            Columns: []clause.Column{
                {Name: "player_id"},
                {Name: "season"},
                {Name: "team"},
                {Name: "is_playoff"},
            },
            DoUpdates: clause.AssignmentColumns([]string{
                "external_id", "player_name", "position", "age", "games", "games_started", 
                "minutes_played",
                "per", "ts_percent", "three_par", "ftr",
                "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
                "assist_percent", "steal_percent", "block_percent", "turnover_percent",
                "usage_percent", "offensive_ws", "defensive_ws", "win_shares",
                "win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
            }),
        }).Create(&stat).Error; err != nil {
            log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
        }
    })

    return nil
}

//         // 3. map into your model
        // stat := models.PlayerAdvancedStat{
//             ExternalID:         mustAtoi(data["rk"]),
//             PlayerID:           playerID,
//             PlayerName:         data["player"],
//             Position:           data["pos"],
//             Age:                mustAtoi(data["age"]),
//             Games:              mustAtoi(data["g"]),
//             MinutesPlayed:      mustAtoi(data["mp"]),
//             PER:                mustParseFloat(data["per"]),
//             TSPercent:          mustParseFloat(data["ts_pct"]),
//             ThreePAR:           mustParseFloat(data["three_par"]),
//             FTR:                mustParseFloat(data["ftr"]),
//             OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
//             DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
//             TotalRBPercent:     mustParseFloat(data["trb_pct"]),
//             AssistPercent:      mustParseFloat(data["ast_pct"]),
//             StealPercent:       mustParseFloat(data["stl_pct"]),
//             BlockPercent:       mustParseFloat(data["blk_pct"]),
//             TurnoverPercent:    mustParseFloat(data["tov_pct"]),
//             UsagePercent:       mustParseFloat(data["usg_pct"]),
//             OffensiveWS:        mustParseFloat(data["ows"]),
//             DefensiveWS:        mustParseFloat(data["dws"]),
//             WinShares:          mustParseFloat(data["ws"]),
//             WinSharesPer:       mustParseFloat(data["ws_per_48"]),
//             OffensiveBox:       mustParseFloat(data["obpm"]),
//             DefensiveBox:       mustParseFloat(data["dbpm"]),
//             Box:                mustParseFloat(data["bpm"]),
//             VORP:               mustParseFloat(data["vorp"]),
//             Team:               data["team_id"],
//             Season:             season,
//             IsPlayoff:          isPlayoff,
//         }

//         // 4. upsert
//         if err := db.Clauses(clause.OnConflict{
//             Columns: []clause.Column{
//                 {Name: "player_id"},
//                 {Name: "season"},
//                 {Name: "team"},
//                 {Name: "is_playoff"},
//             },
//             DoUpdates: clause.AssignmentColumns([]string{
//                 "external_id", "player_name", "position", "age", "games", "minutes_played",
//                 "per", "ts_percent", "three_par", "ftr",
//                 "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
//                 "assist_percent", "steal_percent", "block_percent", "turnover_percent",
//                 "usage_percent", "offensive_ws", "defensive_ws", "win_shares",
//                 "win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
//             }),
//         }).Create(&stat).Error; err != nil {
//             log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
//         }
//     })

//     return nil
// }