From 7ca427b386efec29ed454e1ed5e22b6277375a91 Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Thu, 21 Aug 2025 00:41:36 -0500 Subject: [PATCH] box score scrape service updates batch handle --- .DS_Store | Bin 30724 -> 30724 bytes Dockerfile | 4 +- docker-compose.local.yml | 2 +- import.go | 69 +++++++++++---------- main.go | 13 ++-- services/box_score_scrape_service.go | 89 ++++++++++++++++++++------- 6 files changed, 113 insertions(+), 64 deletions(-) diff --git a/.DS_Store b/.DS_Store index bd3a02a9b143d9e07297e1bceea6afa0adca19e7..146dc2ba2b5a69be25b9817d5376e4ad50ac1016 100644 GIT binary patch delta 761 zcmX|;OH30{6o&g#(K|9Gw>*r4q#YzmgM~sVFEvO3b%9th5Z>z0xynd8g-!txur7!( z#`i3Yx>cjDj1%HBZj2G#O5%ncaUmNpE=|ux%dC()-bn*=Svm&Rn^r# zxu(8m)+bLl1S)-Uur_F|Ra65nt+D(vq}-z1rRgd=-!-YJ$>Cnbh%jB%^t0U)nxPFv z8QtQ7Paum@;m)F6C8A8Tsk$ypyw`<+mow^cZW~i~!jPKdO8qk6^wBtBBa02}p z#3`JHf^&#MM;xP=!Zc=a5h+~3Rb0b$+`wJj!+ku!Lp;V4JjF9un8z!;#v6RVM|{Ep zKI03%VhP`|VxrHTh2Pe4;IL>G`&X=-rXm@Q=1=*XQ!Eh8t%^TIUAcKup;Sa@3F#Rh z(#FQMxsl;F{URC zyBv~Qc%tNFdihbsNQOt5zVWP(bcB#TV^~SB@TATX8xFI$aJ!WGs7DGD(wp{%RF&KE zT)EWj7_9qGCm30F{BZno{Bit67Mvh(As6{5fE%Uoq6~Xcj}V$Uzc6~x%h?TNxQ!sj z$r+g8)Ka*F%eaYKxC1l8>rsZ+3%tcUyf?SC`q6D3i+C_>=@GAB#hX`Q&FV9oEwCKt EKSnmibpQYW delta 1132 zcmb`FUu+ab9LMK7Xgez%nYPrlY`4AiXsO4QmeWcrg*#FjskA}qwRb&?xV_!BTg#r_ zZvOz13#LY6)BtNV@kPM+<{$Jzf*2Fg7b65CIek!rF+Pxp#`3^}k{D-i%Y!lT$$i+_ z-}k#SznS@dmX@)!j59xMS{v)#91X>KdX7XxhkIjP(NOo6-s1A6```_&zo(Ahtqne0 zHMpA(XY|FSx#*5qhfEJM3Gb)+I`4yhZQjvTZB5$B=4{^yCnBW$c*kd^3r#$fnztNl zV%8j8unc#4Tz9gjZCLiysAU`GBCAGJ5tXHeb!wv)=B-Q^inuc<+j)eE-HNCP$_}lQ ziL*@5a>--b{ff|;SQV5VT93j6f?_M*rU*w7g*!@(>(qvLf^$Kt@w=h~OATlbG4TZx zjD>^E60-L8$I6Ju4Ba({bA~l(nT8ThbgdYSG?NJFBRff&%#Z?kfxJQ9Cg;cn@+G-Q zu99oyH*ym|1!|G2hlVJ+u?_tg#t4!~A&m^OFffTJSjfY{9Okizqj(lecn;6w7>>ik zt9T7>;v`PtExdzw@jlMrLwt-+@F_0fGhDOTKY`}3xJ8!Hnww5vSnNN{A9!H>}n z&XZ3{I%I-@vN^`p(OG@MoRxIQJ_Xt6fh}B3&6|10%GnZ5jYSDbqF$nSS66NH60|>v?m}f^Ftk)Iq847i-G8KWV_Q+}v$*xFC-* z?ytAQFvyodVw30Ha&>Gs%NHy9-H_&z(p&RLT8mGJHIGv 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})), - }).Create(&pbs).Error; err != nil { - return fmt.Errorf("failed to upsert player basic stats: %w", err) + for i := 0; i < len(pbs); i += batchSize { + end := i + batchSize + if end > len(pbs) { + end = len(pbs) + } + batch := pbs[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameBasicStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert player basic stats batch: %w", err) + } } + log.Printf("Successfully upserted %d player basic stats.", len(pbs)) } + + // Batch upsert Player Advanced Stats if len(pas) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})), - }).Create(&pas).Error; err != nil { - return fmt.Errorf("failed to upsert player advanced stats: %w", err) + for i := 0; i < len(pas); i += batchSize { + end := i + batchSize + if end > len(pas) { + end = len(pas) + } + batch := pas[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "player_id"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.PlayerGameAdvStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert player advanced stats batch: %w", err) + } } + log.Printf("Successfully upserted %d player advanced stats.", len(pas)) } + + // Batch upsert Team Basic Stats if len(tbs) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})), - }).Create(&tbs).Error; err != nil { - return fmt.Errorf("failed to upsert team basic stats: %w", err) + for i := 0; i < len(tbs); i += batchSize { + end := i + batchSize + if end > len(tbs) { + end = len(tbs) + } + batch := tbs[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameBasicStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert team basic stats batch: %w", err) + } } + log.Printf("Successfully upserted %d team basic stats.", len(tbs)) } + + // Batch upsert Team Advanced Stats if len(tas) > 0 { - if err := db.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, - DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})), - }).Create(&tas).Error; err != nil { - return fmt.Errorf("failed to upsert team advanced stats: %w", err) + for i := 0; i < len(tas); i += batchSize { + end := i + batchSize + if end > len(tas) { + end = len(tas) + } + batch := tas[i:end] + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "game_id"}, {Name: "team"}}, + DoUpdates: clause.AssignmentColumns(getModelColumns(&models.TeamGameAdvStat{})), + }).Create(&batch).Error; err != nil { + return fmt.Errorf("failed to upsert team advanced stats batch: %w", err) + } } + log.Printf("Successfully upserted %d team advanced stats.", len(tas)) } + return nil } \ No newline at end of file