nprasad2077/NBA_Go
1
0
Fork 0
mirror of https://github.com/nprasad2077/NBA_Go.git synced 2026-09-22 05:55:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fields fixed

Browse Source
This commit is contained in:
Ravi Prasad
2025-06-03 02:25:32 -05:00
parent 58b27fdb76
commit e9e77e8e22
8 changed files with 213 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.DS_Store
Vendored
BIN
View File
Binary file not shown.
grafana/.DS_Store
Vendored
BIN
View File
Binary file not shown.
grafana/provisioning/.DS_Store
Vendored
BIN
View File
Binary file not shown.
import.go
+5 -6
View File
@@ -9,9 +9,9 @@ import (
"github.com/nprasad2077/NBA_Go/utils"
)
// importPlayerAdvanced fetches and stores advanced stats for seasons 2023–2025
// importPlayerAdvanced fetches and stores advanced stats for seasons 2017–2025
func importPlayerAdvanced(db *gorm.DB) {
for season := 2023; season <= 2025; season++ {
for season := 2017; season <= 2022; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, false); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
@@ -22,9 +22,8 @@ func importPlayerAdvanced(db *gorm.DB) {
}
// importPlayerAdvancedPlayoffs fetches and stores advanced stats for playoffs seasons 2023–2025
func importPlayerAdvancedPlayoffs(db *gorm.DB) {
for season := 2023; season <= 2025; season++ {
for season := 2017; season <= 2022; season++ {
if err := services.FetchAndStorePlayerAdvancedScrapedStats(db, season, true); err != nil {
log.Printf("advanced import failed for %d: %v", season, err)
}
@@ -46,7 +45,7 @@ func importPlayerShotChart(db *gorm.DB) {
// importPlayerTotalsScrape fetches & stores scraped regular-season total stats
func importPlayerTotalsScrape(db *gorm.DB) {
for season := 1993; season <= 2025; season++ {
for season := 2017; season <= 2022; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, false); err != nil {
log.Printf("scraped totals import failed for %d: %v", season, err)
}
@@ -58,7 +57,7 @@ func importPlayerTotalsScrape(db *gorm.DB) {
// importPlayerPlayoffsScrape fetches & stores scraped playoff total stats
func importPlayerTotalsPlayoffsScrape(db *gorm.DB) {
for season := 1993; season <= 2025; season++ {
for season := 2017; season <= 2022; season++ {
if err := services.FetchAndStorePlayerTotalScrapedStats(db, season, true); err != nil {
log.Printf("scraped playoffs import failed for %d: %v", season, err)
}
main.go
+4
View File
@@ -21,6 +21,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/adaptor"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/prometheus/client_golang/prometheus/promhttp"
fiberswagger "github.com/swaggo/fiber-swagger"
@@ -73,6 +74,9 @@ func main() {
},
})
// — CORS Allow ALL origins (development) —
app.Use(cors.New())
// middlewares
app.Use(logger.New())
app.Use(middleware.MetricsMiddleware())
models/player_advanced_stat.go
+1
View File
@@ -14,6 +14,7 @@ type PlayerAdvancedStat struct {
Position string `json:"position"`
Age int `json:"age"`
Games int `json:"games"`
GamesStarted int `json:"gamesStarted"`
MinutesPlayed int `json:"minutesPlayed"`
PER float64 `json:"per"`
TSPercent float64 `json:"tsPercent"`
services/player_advanced_playoffs_scrape_service.go
+88 -10
View File
@@ -54,7 +54,7 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
return err
}
// 📌 pick the right table selector
// pick the right table selector
var table *goquery.Selection
if isPlayoff {
table = doc.Find("#div_advanced_stats table#advanced_stats")
@@ -96,19 +96,38 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
}
data["player-additional"] = playerID
// 3. map into your model
// 4) pick the correct keys for ExternalID, PlayerName, Team, depending on season vs. playoff
// - playoffs tables still use "rk", "player", "team_id"
// - regular‐season advanced uses "ranker", "name_display", "team_name_abbr"
extID := mustAtoi(data["rk"])
if extID == 0 {
extID = mustAtoi(data["ranker"])
}
playerName := data["player"]
if playerName == "" {
playerName = data["name_display"]
}
teamID := data["team_id"]
if teamID == "" {
teamID = data["team_name_abbr"]
}
// 5) map into your GORM model
stat := models.PlayerAdvancedStat{
ExternalID: mustAtoi(data["rk"]),
ExternalID: extID,
PlayerID: playerID,
PlayerName: data["player"],
PlayerName: playerName,
Position: data["pos"],
Age: mustAtoi(data["age"]),
Games: mustAtoi(data["g"]),
GamesStarted: mustAtoi(data["games_started"]),
MinutesPlayed: mustAtoi(data["mp"]),
PER: mustParseFloat(data["per"]),
TSPercent: mustParseFloat(data["ts_pct"]),
ThreePAR: mustParseFloat(data["three_par"]),
FTR: mustParseFloat(data["ftr"]),
ThreePAR: mustParseFloat(data["fg3a_per_fga_pct"]),
FTR: mustParseFloat(data["fta_per_fga_pct"]),
OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
TotalRBPercent: mustParseFloat(data["trb_pct"]),
@@ -125,12 +144,12 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
DefensiveBox: mustParseFloat(data["dbpm"]),
Box: mustParseFloat(data["bpm"]),
VORP: mustParseFloat(data["vorp"]),
Team: data["team_id"],
Team: teamID,
Season: season,
IsPlayoff: isPlayoff,
}
// 4. upsert
// 6) upsert on (player_id, season, team, is_playoff)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "player_id"},
@@ -139,7 +158,8 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
{Name: "is_playoff"},
},
DoUpdates: clause.AssignmentColumns([]string{
"external_id", "player_name", "position", "age", "games", "minutes_played",
"external_id", "player_name", "position", "age", "games", "games_started",
"minutes_played",
"per", "ts_percent", "three_par", "ftr",
"offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
"assist_percent", "steal_percent", "block_percent", "turnover_percent",
@@ -152,4 +172,62 @@ func FetchAndStorePlayerAdvancedScrapedStats(db *gorm.DB, season int, isPlayoff
})
return nil
}
}
// // 3. map into your model
// stat := models.PlayerAdvancedStat{
// ExternalID: mustAtoi(data["rk"]),
// PlayerID: playerID,
// PlayerName: data["player"],
// Position: data["pos"],
// Age: mustAtoi(data["age"]),
// Games: mustAtoi(data["g"]),
// MinutesPlayed: mustAtoi(data["mp"]),
// PER: mustParseFloat(data["per"]),
// TSPercent: mustParseFloat(data["ts_pct"]),
// ThreePAR: mustParseFloat(data["three_par"]),
// FTR: mustParseFloat(data["ftr"]),
// OffensiveRBPercent: mustParseFloat(data["orb_pct"]),
// DefensiveRBPercent: mustParseFloat(data["drb_pct"]),
// TotalRBPercent: mustParseFloat(data["trb_pct"]),
// AssistPercent: mustParseFloat(data["ast_pct"]),
// StealPercent: mustParseFloat(data["stl_pct"]),
// BlockPercent: mustParseFloat(data["blk_pct"]),
// TurnoverPercent: mustParseFloat(data["tov_pct"]),
// UsagePercent: mustParseFloat(data["usg_pct"]),
// OffensiveWS: mustParseFloat(data["ows"]),
// DefensiveWS: mustParseFloat(data["dws"]),
// WinShares: mustParseFloat(data["ws"]),
// WinSharesPer: mustParseFloat(data["ws_per_48"]),
// OffensiveBox: mustParseFloat(data["obpm"]),
// DefensiveBox: mustParseFloat(data["dbpm"]),
// Box: mustParseFloat(data["bpm"]),
// VORP: mustParseFloat(data["vorp"]),
// Team: data["team_id"],
// Season: season,
// IsPlayoff: isPlayoff,
// }
// // 4. upsert
// if err := db.Clauses(clause.OnConflict{
// Columns: []clause.Column{
// {Name: "player_id"},
// {Name: "season"},
// {Name: "team"},
// {Name: "is_playoff"},
// },
// DoUpdates: clause.AssignmentColumns([]string{
// "external_id", "player_name", "position", "age", "games", "minutes_played",
// "per", "ts_percent", "three_par", "ftr",
// "offensive_rb_percent", "defensive_rb_percent", "total_rb_percent",
// "assist_percent", "steal_percent", "block_percent", "turnover_percent",
// "usage_percent", "offensive_ws", "defensive_ws", "win_shares",
// "win_shares_per", "offensive_box", "defensive_box", "box", "vorp",
// }),
// }).Create(&stat).Error; err != nil {
// log.Printf("Failed upsert advanced for %s: %v", stat.PlayerID, err)
// }
// })
// return nil
// }
services/player_total_scrape_service.go
+115 -37
View File
@@ -46,7 +46,7 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
return err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
@@ -59,6 +59,30 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
return fmt.Errorf("could not find table#totals_stats")
}
// table := doc.Find("table#totals_stats")
// if table.Length() == 0 {
// commentSel := doc.
// Find("div#all_totals_stats").
// Contents().
// FilterFunction(func(i int, s *goquery.Selection) bool {
// return goquery.NodeName(s) == "#comment"
// })
// if commentSel.Length() == 0 {
// return fmt.Errorf("could not find table#totals_stats (even inside comment)")
// }
// commentedHTML := commentSel.Nodes[0].FirstChild.Data
// innerDoc, err := goquery.NewDocumentFromReader(strings.NewReader(commentedHTML))
// if err != nil {
// return fmt.Errorf("failed to parse commented totals_stats HTML: %w", err)
// }
// table = innerDoc.Find("table#totals_stats")
// if table.Length() == 0 {
// return fmt.Errorf("could not find table#totals_stats after un‐commenting")
// }
// }
// 1) collect the data-stat keys in header order
var headers []string
table.Find("thead tr th").Each(func(i int, th *goquery.Selection) {
@@ -92,42 +116,96 @@ func FetchAndStorePlayerTotalScrapedStats(db *gorm.DB, season int, isPlayoff boo
}
data["player-additional"] = playerID
// 3) map into GORM model
stat := models.PlayerTotalStat{
ExternalID: mustAtoi(data["rk"]),
PlayerID: playerID,
PlayerName: data["player"],
Position: data["pos"],
Age: mustAtoi(data["age"]),
Games: mustAtoi(data["g"]),
GamesStarted: mustAtoi(data["gs"]),
MinutesPG: mustParseFloat(data["mp"]), // TOTAL minutes as float
FieldGoals: mustAtoi(data["fg"]),
FieldAttempts: mustAtoi(data["fga"]),
FieldPercent: mustParseFloat(data["fg_pct"]),
ThreeFG: mustAtoi(data["fg3"]),
ThreeAttempts: mustAtoi(data["fg3a"]),
ThreePercent: mustParseFloat(data["fg3_pct"]),
TwoFG: mustAtoi(data["fg2"]),
TwoAttempts: mustAtoi(data["fg2a"]),
TwoPercent: mustParseFloat(data["fg2_pct"]),
EffectFGPercent: mustParseFloat(data["efg_pct"]),
FT: mustAtoi(data["ft"]),
FTAttempts: mustAtoi(data["fta"]),
FTPercent: mustParseFloat(data["ft_pct"]),
OffensiveRB: mustAtoi(data["orb"]),
DefensiveRB: mustAtoi(data["drb"]),
TotalRB: mustAtoi(data["trb"]),
Assists: mustAtoi(data["ast"]),
Steals: mustAtoi(data["stl"]),
Blocks: mustAtoi(data["blk"]),
Turnovers: mustAtoi(data["tov"]),
PersonalFouls: mustAtoi(data["pf"]),
Points: mustAtoi(data["pts"]),
Team: data["team_id"],
Season: season,
IsPlayoff: isPlayoff,
}
// 3a) pick the right “ExternalID” key (playoffs use “rk”; season uses “ranker”)
extID := mustAtoi(data["rk"])
if extID == 0 {
extID = mustAtoi(data["ranker"])
}
// 3b) pick the right “PlayerName” key (playoffs use “player”; season uses “name_display”)
playerName := data["player"]
if playerName == "" {
playerName = data["name_display"]
}
// 3c) pick the right “Team” key (playoffs use “team_id”; season uses “team_name_abbr”)
teamID := data["team_id"]
if teamID == "" {
teamID = data["team_name_abbr"]
}
stat := models.PlayerTotalStat{
ExternalID: extID,
PlayerID: playerID,
PlayerName: playerName,
Position: data["pos"],
Age: mustAtoi(data["age"]),
Games: mustAtoi(data["games"]),
GamesStarted: mustAtoi(data["games_started"]),
MinutesPG: mustParseFloat(data["mp"]),
FieldGoals: mustAtoi(data["fg"]),
FieldAttempts: mustAtoi(data["fga"]),
FieldPercent: mustParseFloat(data["fg_pct"]),
ThreeFG: mustAtoi(data["fg3"]),
ThreeAttempts: mustAtoi(data["fg3a"]),
ThreePercent: mustParseFloat(data["fg3_pct"]),
TwoFG: mustAtoi(data["fg2"]),
TwoAttempts: mustAtoi(data["fg2a"]),
TwoPercent: mustParseFloat(data["fg2_pct"]),
EffectFGPercent: mustParseFloat(data["efg_pct"]),
FT: mustAtoi(data["ft"]),
FTAttempts: mustAtoi(data["fta"]),
FTPercent: mustParseFloat(data["ft_pct"]),
OffensiveRB: mustAtoi(data["orb"]),
DefensiveRB: mustAtoi(data["drb"]),
TotalRB: mustAtoi(data["trb"]),
Assists: mustAtoi(data["ast"]),
Steals: mustAtoi(data["stl"]),
Blocks: mustAtoi(data["blk"]),
Turnovers: mustAtoi(data["tov"]),
PersonalFouls: mustAtoi(data["pf"]),
Points: mustAtoi(data["pts"]),
Team: teamID,
Season: season,
IsPlayoff: isPlayoff,
}
// // 3) map into GORM model
// stat := models.PlayerTotalStat{
// ExternalID: mustAtoi(data["rk"]),
// PlayerID: playerID,
// PlayerName: data["player"],
// Position: data["pos"],
// Age: mustAtoi(data["age"]),
// Games: mustAtoi(data["g"]),
// GamesStarted: mustAtoi(data["gs"]),
// MinutesPG: mustParseFloat(data["mp"]), // TOTAL minutes as float
// FieldGoals: mustAtoi(data["fg"]),
// FieldAttempts: mustAtoi(data["fga"]),
// FieldPercent: mustParseFloat(data["fg_pct"]),
// ThreeFG: mustAtoi(data["fg3"]),
// ThreeAttempts: mustAtoi(data["fg3a"]),
// ThreePercent: mustParseFloat(data["fg3_pct"]),
// TwoFG: mustAtoi(data["fg2"]),
// TwoAttempts: mustAtoi(data["fg2a"]),
// TwoPercent: mustParseFloat(data["fg2_pct"]),
// EffectFGPercent: mustParseFloat(data["efg_pct"]),
// FT: mustAtoi(data["ft"]),
// FTAttempts: mustAtoi(data["fta"]),
// FTPercent: mustParseFloat(data["ft_pct"]),
// OffensiveRB: mustAtoi(data["orb"]),
// DefensiveRB: mustAtoi(data["drb"]),
// TotalRB: mustAtoi(data["trb"]),
// Assists: mustAtoi(data["ast"]),
// Steals: mustAtoi(data["stl"]),
// Blocks: mustAtoi(data["blk"]),
// Turnovers: mustAtoi(data["tov"]),
// PersonalFouls: mustAtoi(data["pf"]),
// Points: mustAtoi(data["pts"]),
// Team: data["team_id"],
// Season: season,
// IsPlayoff: isPlayoff,
// }
// 4) upsert on (player_id, season, team, is_playoff)
if err := db.Clauses(clause.OnConflict{