diff --git a/.DS_Store b/.DS_Store index 678b385..f79fcc8 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/grafana/.DS_Store b/grafana/.DS_Store new file mode 100644 index 0000000..5c7c175 Binary files /dev/null and b/grafana/.DS_Store differ diff --git a/grafana/provisioning/.DS_Store b/grafana/provisioning/.DS_Store new file mode 100644 index 0000000..29c24ed Binary files /dev/null and b/grafana/provisioning/.DS_Store differ diff --git a/import.go b/import.go index 857e658..9c1554f 100644 --- a/import.go +++ b/import.go @@ -9,9 +9,9 @@ import ( "github.com/nprasad2077/NBA_Go/utils" ) -// importPlayerAdvanced fetches and stores advanced stats for seasons 2023–2025 +// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025 func importPlayerAdvanced(db *gorm.DB) { - for season := 2023; season <= 2025; season++ { + for season := 2017; season <= 2022; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -22,9 +22,8 @@ func importPlayerAdvanced(db *gorm.DB) { } // importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025 - func importPlayerAdvancedPlayoffs(db *gorm.DB) { - for season := 2023; season <= 2025; season++ { + for season := 2017; season <= 2022; season++ { if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil { log.Printf("advanced import failed for %d: %v", season, err) } @@ -46,7 +45,7 @@ func importPlayerShotChart(db *gorm.DB) { // importPlayerTotalsScrape fetches & stores scraped regular-season total stats func importPlayerTotalsScrape(db *gorm.DB) { - for season := 1993; season <= 2025; season++ { + for season := 2017; season <= 2022; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil { log.Printf("scraped totals import failed for %d: %v", season, err) } @@ -58,7 +57,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { // importPlayerPlayoffsScrape fetches & stores scraped playoff total stats func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { - for season := 1993; season <= 2025; season++ { + for season := 2017; season <= 2022; season++ { if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil { log.Printf("scraped playoffs import failed for %d: %v", season, err) } diff --git a/main.go b/main.go index 1243581..a1602e7 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/adaptor" + "github.com/gofiber/fiber/v2/middleware/cors" "github.com/gofiber/fiber/v2/middleware/logger" "github.com/prometheus/client_golang/prometheus/promhttp" fiberswagger "github.com/swaggo/fiber-swagger" @@ -73,6 +74,9 @@ func main() { }, }) + // — CORS Allow ALL origins (development) — + app.Use(cors.New()) + // middlewares app.Use(logger.New()) app.Use(middleware.MetricsMiddleware()) diff --git a/models/player_advanced_stat.go b/models/player_advanced_stat.go index bb79a7e..13c4010 100644 --- a/models/player_advanced_stat.go +++ b/models/player_advanced_stat.go @@ -14,6 +14,7 @@ type PlayerAdvancedStat struct { Position string `json:"position"` Age int `json:"age"` Games int `json:"games"` + GamesStarted int `json:"gamesStarted"` MinutesPlayed int `json:"minutesPlayed"` PER float64 `json:"per"` TSPercent float64 `json:"tsPercent"` diff --git a/services/player_advanced_playoffs_scrape_service.go b/services/player_advanced_playoffs_scrape_service.go index 00e0e04..94f40e7 100644 --- a/services/player_advanced_playoffs_scrape_service.go +++ b/services/player_advanced_playoffs_scrape_service.go @@ -54,7 +54,7 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff return err } - // 📌 pick the right table selector + // pick the right table selector var table *goquery.Selection if isPlayoff { table = doc.Find("#div_advanced_stats table#advanced_stats") @@ -96,19 +96,38 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff } data["player-additional"] = playerID - // 3. map into your model + // 4) pick the correct keys for ExternalID, PlayerName, Team, depending on season vs. playoff + // - playoffs tables still use "rk", "player", "team_id" + // - regular‐season advanced uses "ranker", "name_display", "team_name_abbr" + extID := mustAtoi(data["rk"]) + if extID == 0 { + extID = mustAtoi(data["ranker"]) + } + + playerName := data["player"] + if playerName == "" { + playerName = data["name_display"] + } + + teamID := data["team_id"] + if teamID == "" { + teamID = data["team_name_abbr"] + } + + // 5) map into your GORM model stat := models.PlayerAdvancedStat{ - ExternalID: mustAtoi(data["rk"]), + ExternalID: extID, PlayerID: playerID, - PlayerName: data["player"], + PlayerName: playerName, Position: data["pos"], Age: mustAtoi(data["age"]), Games: mustAtoi(data["g"]), + GamesStarted: mustAtoi(data["games_started"]), MinutesPlayed: mustAtoi(data["mp"]), PER: mustParseFloat(data["per"]), TSPercent: mustParseFloat(data["ts_pct"]), - ThreePAR: mustParseFloat(data["three_par"]), - FTR: mustParseFloat(data["ftr"]), + ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]), + FTR: mustParseFloat(data["fta_per_fga_pct"]), OffensiveRBPercent: mustParseFloat(data["orb_pct"]), DefensiveRBPercent: mustParseFloat(data["drb_pct"]), TotalRBPercent: mustParseFloat(data["trb_pct"]), @@ -125,12 +144,12 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff DefensiveBox: mustParseFloat(data["dbpm"]), Box: mustParseFloat(data["bpm"]), VORP: mustParseFloat(data["vorp"]), - Team: data["team_id"], + Team: teamID, Season: season, IsPlayoff: isPlayoff, } - // 4. upsert + // 6) upsert on (player_id, season, team, is_playoff) if err := db.Clauses(clause.OnConflict{ Columns: []clause.Column{ {Name: "player_id"}, @@ -139,7 +158,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff {Name: "is_playoff"}, }, DoUpdates: clause.AssignmentColumns([]string{ - "external_id", "player_name", "position", "age", "games", "minutes_played", + "external_id", "player_name", "position", "age", "games", "games_started", + "minutes_played", "per", "ts_percent", "three_par", "ftr", "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent", "assist_percent", "steal_percent", "block_percent", "turnover_percent", @@ -152,4 +172,62 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff }) return nil -} \ No newline at end of file +} + +// // 3. map into your model + // stat := models.PlayerAdvancedStat{ +// ExternalID: mustAtoi(data["rk"]), +// PlayerID: playerID, +// PlayerName: data["player"], +// Position: data["pos"], +// Age: mustAtoi(data["age"]), +// Games: mustAtoi(data["g"]), +// MinutesPlayed: mustAtoi(data["mp"]), +// PER: mustParseFloat(data["per"]), +// TSPercent: mustParseFloat(data["ts_pct"]), +// ThreePAR: mustParseFloat(data["three_par"]), +// FTR: mustParseFloat(data["ftr"]), +// OffensiveRBPercent: mustParseFloat(data["orb_pct"]), +// DefensiveRBPercent: mustParseFloat(data["drb_pct"]), +// TotalRBPercent: mustParseFloat(data["trb_pct"]), +// AssistPercent: mustParseFloat(data["ast_pct"]), +// StealPercent: mustParseFloat(data["stl_pct"]), +// BlockPercent: mustParseFloat(data["blk_pct"]), +// TurnoverPercent: mustParseFloat(data["tov_pct"]), +// UsagePercent: mustParseFloat(data["usg_pct"]), +// OffensiveWS: mustParseFloat(data["ows"]), +// DefensiveWS: mustParseFloat(data["dws"]), +// WinShares: mustParseFloat(data["ws"]), +// WinSharesPer: mustParseFloat(data["ws_per_48"]), +// OffensiveBox: mustParseFloat(data["obpm"]), +// DefensiveBox: mustParseFloat(data["dbpm"]), +// Box: mustParseFloat(data["bpm"]), +// VORP: mustParseFloat(data["vorp"]), +// Team: data["team_id"], +// Season: season, +// IsPlayoff: isPlayoff, +// } + +// // 4. upsert +// if err := db.Clauses(clause.OnConflict{ +// Columns: []clause.Column{ +// {Name: "player_id"}, +// {Name: "season"}, +// {Name: "team"}, +// {Name: "is_playoff"}, +// }, +// DoUpdates: clause.AssignmentColumns([]string{ +// "external_id", "player_name", "position", "age", "games", "minutes_played", +// "per", "ts_percent", "three_par", "ftr", +// "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent", +// "assist_percent", "steal_percent", "block_percent", "turnover_percent", +// "usage_percent", "offensive_ws", "defensive_ws", "win_shares", +// "win_shares_per", "offensive_box", "defensive_box", "box", "vorp", +// }), +// }).Create(&stat).Error; err != nil { +// log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err) +// } +// }) + +// return nil +// } \ No newline at end of file diff --git a/services/player_total_scrape_service.go b/services/player_total_scrape_service.go index 5f9e08f..f199c9a 100644 --- a/services/player_total_scrape_service.go +++ b/services/player_total_scrape_service.go @@ -46,7 +46,7 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo body, err := io.ReadAll(resp.Body) if err != nil { - return err + return err } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) @@ -59,6 +59,30 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo return fmt.Errorf("could not find table#totals_stats") } + // table := doc.Find("table#totals_stats") + // if table.Length() == 0 { + // commentSel := doc. + // Find("div#all_totals_stats"). + // Contents(). + // FilterFunction(func(i int, s *goquery.Selection) bool { + // return goquery.NodeName(s) == "#comment" + // }) + + // if commentSel.Length() == 0 { + // return fmt.Errorf("could not find table#totals_stats (even inside comment)") + // } + + // commentedHTML := commentSel.Nodes[0].FirstChild.Data + // innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML)) + // if err != nil { + // return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err) + // } + // table = innerDoc.Find("table#totals_stats") + // if table.Length() == 0 { + // return fmt.Errorf("could not find table#totals_stats after un‐commenting") + // } + // } + // 1) collect the data-stat keys in header order var headers []string table.Find("thead tr th").Each(func(i int, th *goquery.Selection) { @@ -92,42 +116,96 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo } data["player-additional"] = playerID - // 3) map into GORM model - stat := models.PlayerTotalStat{ - ExternalID: mustAtoi(data["rk"]), - PlayerID: playerID, - PlayerName: data["player"], - Position: data["pos"], - Age: mustAtoi(data["age"]), - Games: mustAtoi(data["g"]), - GamesStarted: mustAtoi(data["gs"]), - MinutesPG: mustParseFloat(data["mp"]), // TOTAL minutes as float - FieldGoals: mustAtoi(data["fg"]), - FieldAttempts: mustAtoi(data["fga"]), - FieldPercent: mustParseFloat(data["fg_pct"]), - ThreeFG: mustAtoi(data["fg3"]), - ThreeAttempts: mustAtoi(data["fg3a"]), - ThreePercent: mustParseFloat(data["fg3_pct"]), - TwoFG: mustAtoi(data["fg2"]), - TwoAttempts: mustAtoi(data["fg2a"]), - TwoPercent: mustParseFloat(data["fg2_pct"]), - EffectFGPercent: mustParseFloat(data["efg_pct"]), - FT: mustAtoi(data["ft"]), - FTAttempts: mustAtoi(data["fta"]), - FTPercent: mustParseFloat(data["ft_pct"]), - OffensiveRB: mustAtoi(data["orb"]), - DefensiveRB: mustAtoi(data["drb"]), - TotalRB: mustAtoi(data["trb"]), - Assists: mustAtoi(data["ast"]), - Steals: mustAtoi(data["stl"]), - Blocks: mustAtoi(data["blk"]), - Turnovers: mustAtoi(data["tov"]), - PersonalFouls: mustAtoi(data["pf"]), - Points: mustAtoi(data["pts"]), - Team: data["team_id"], - Season: season, - IsPlayoff: isPlayoff, - } + // 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”) + extID := mustAtoi(data["rk"]) + if extID == 0 { + extID = mustAtoi(data["ranker"]) + } + + // 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”) + playerName := data["player"] + if playerName == "" { + playerName = data["name_display"] + } + + // 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”) + teamID := data["team_id"] + if teamID == "" { + teamID = data["team_name_abbr"] + } + + stat := models.PlayerTotalStat{ + ExternalID: extID, + PlayerID: playerID, + PlayerName: playerName, + Position: data["pos"], + Age: mustAtoi(data["age"]), + Games: mustAtoi(data["games"]), + GamesStarted: mustAtoi(data["games_started"]), + MinutesPG: mustParseFloat(data["mp"]), + FieldGoals: mustAtoi(data["fg"]), + FieldAttempts: mustAtoi(data["fga"]), + FieldPercent: mustParseFloat(data["fg_pct"]), + ThreeFG: mustAtoi(data["fg3"]), + ThreeAttempts: mustAtoi(data["fg3a"]), + ThreePercent: mustParseFloat(data["fg3_pct"]), + TwoFG: mustAtoi(data["fg2"]), + TwoAttempts: mustAtoi(data["fg2a"]), + TwoPercent: mustParseFloat(data["fg2_pct"]), + EffectFGPercent: mustParseFloat(data["efg_pct"]), + FT: mustAtoi(data["ft"]), + FTAttempts: mustAtoi(data["fta"]), + FTPercent: mustParseFloat(data["ft_pct"]), + OffensiveRB: mustAtoi(data["orb"]), + DefensiveRB: mustAtoi(data["drb"]), + TotalRB: mustAtoi(data["trb"]), + Assists: mustAtoi(data["ast"]), + Steals: mustAtoi(data["stl"]), + Blocks: mustAtoi(data["blk"]), + Turnovers: mustAtoi(data["tov"]), + PersonalFouls: mustAtoi(data["pf"]), + Points: mustAtoi(data["pts"]), + Team: teamID, + Season: season, + IsPlayoff: isPlayoff, + } + + // // 3) map into GORM model + // stat := models.PlayerTotalStat{ + // ExternalID: mustAtoi(data["rk"]), + // PlayerID: playerID, + // PlayerName: data["player"], + // Position: data["pos"], + // Age: mustAtoi(data["age"]), + // Games: mustAtoi(data["g"]), + // GamesStarted: mustAtoi(data["gs"]), + // MinutesPG: mustParseFloat(data["mp"]), // TOTAL minutes as float + // FieldGoals: mustAtoi(data["fg"]), + // FieldAttempts: mustAtoi(data["fga"]), + // FieldPercent: mustParseFloat(data["fg_pct"]), + // ThreeFG: mustAtoi(data["fg3"]), + // ThreeAttempts: mustAtoi(data["fg3a"]), + // ThreePercent: mustParseFloat(data["fg3_pct"]), + // TwoFG: mustAtoi(data["fg2"]), + // TwoAttempts: mustAtoi(data["fg2a"]), + // TwoPercent: mustParseFloat(data["fg2_pct"]), + // EffectFGPercent: mustParseFloat(data["efg_pct"]), + // FT: mustAtoi(data["ft"]), + // FTAttempts: mustAtoi(data["fta"]), + // FTPercent: mustParseFloat(data["ft_pct"]), + // OffensiveRB: mustAtoi(data["orb"]), + // DefensiveRB: mustAtoi(data["drb"]), + // TotalRB: mustAtoi(data["trb"]), + // Assists: mustAtoi(data["ast"]), + // Steals: mustAtoi(data["stl"]), + // Blocks: mustAtoi(data["blk"]), + // Turnovers: mustAtoi(data["tov"]), + // PersonalFouls: mustAtoi(data["pf"]), + // Points: mustAtoi(data["pts"]), + // Team: data["team_id"], + // Season: season, + // IsPlayoff: isPlayoff, + // } // 4) upsert on (player_id, season, team, is_playoff) if err := db.Clauses(clause.OnConflict{