From 8973bb18e1dd2f97fb7e545c2104534919802b4e Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Fri, 20 Jun 2025 20:29:13 -0500 Subject: [PATCH 1/2] shot chart batch fix --- services/player_shot_chart_scrape_service.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/services/player_shot_chart_scrape_service.go b/services/player_shot_chart_scrape_service.go index 42c973a..cefa62c 100644 --- a/services/player_shot_chart_scrape_service.go +++ b/services/player_shot_chart_scrape_service.go @@ -85,6 +85,9 @@ func FetchAndStoreShotChartScrapedForPlayer( // --- BATCHING LOGIC START --- // Create a slice to hold all the shot data for the current season. var shotsToUpsert []models.PlayerShotChart + // >>>>>>>>>> FIX START: Add a map to track unique shots to prevent duplicates. + uniqueShots := make(map[string]struct{}) + // <<<<<<<<<< FIX END // 4) Scrape every tooltip and collect the data into the slice. wrapper.Find("div.tooltip.make, div.tooltip.miss").Each(func(_ int, s *goquery.Selection) { @@ -130,6 +133,18 @@ func FetchAndStoreShotChartScrapedForPlayer( teamScore, oppScore := mustAtoi(sc[0]), mustAtoi(sc[1]) lead := teamScore > oppScore + // >>>>>>>>>> FIX START: Create a unique key based on the conflict columns. + uniqueKey := fmt.Sprintf("%s|%d|%s|%s|%s|%d|%d", + playerID, season, date, quarter, timeRem, top, left) + + // If we have already seen this key, skip this iteration. + if _, exists := uniqueShots[uniqueKey]; exists { + return + } + uniqueShots[uniqueKey] = struct{}{} + // <<<<<<<<<< FIX END + + shot := models.PlayerShotChart{ PlayerID: playerID, PlayerName: playerName, @@ -156,6 +171,8 @@ func FetchAndStoreShotChartScrapedForPlayer( if len(shotsToUpsert) > 0 { log.Printf("Attempting to batch upsert %d shots for player %s in season %d...", len(shotsToUpsert), playerID, season) + // NOTE: I am assuming the column name for the "Quarter" field is "qtr". + // If not, you must update the clause.OnConflict below. if err := db.Clauses(clause.OnConflict{ Columns: []clause.Column{ // MUST match the unique index order in the model {Name: "player_id"}, {Name: "season"}, {Name: "date"}, @@ -183,6 +200,7 @@ func FetchAndStoreShotChartScrapedForPlayer( // extractCommentedShotChart returns the inner HTML of the comment block that // contains
. BR hides the SVG there for ad reasons. func extractCommentedShotChart(htmlBytes []byte) string { + // ... (this function remains unchanged) root, err := html.Parse(bytes.NewReader(htmlBytes)) if err != nil { return "" @@ -204,6 +222,7 @@ func extractCommentedShotChart(htmlBytes []byte) string { // parsePx turns "left:244px" or "top:18px" into int(244 / 18). func parsePx(s string) int { + // ... (this function remains unchanged) parts := strings.Split(s, ":") return mustAtoi(strings.TrimSuffix(parts[1], "px")) } \ No newline at end of file From 42e858bef545f53f610903c21621f0bbe33c348e Mon Sep 17 00:00:00 2001 From: Ravi Prasad Date: Fri, 20 Jun 2025 20:37:42 -0500 Subject: [PATCH 2/2] notes --- docs/notes/redis.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/notes/redis.md diff --git a/docs/notes/redis.md b/docs/notes/redis.md new file mode 100644 index 0000000..2bf231e --- /dev/null +++ b/docs/notes/redis.md @@ -0,0 +1,10 @@ +# Implement a Caching Layer with Redis + +For even more speed, especially for data that doesn't change every second, a caching layer is the answer. The goal is to avoid hitting the database at all for repeated requests. + +* **How it Works:** When a request comes in (e.g., for Houston Rockets 2024 stats), your Go application first checks if the result is already in Redis (an extremely fast in-memory database). + * If it's in Redis, it returns the cached data immediately (e.g., in <50ms). + * If it's not in Redis, it queries the PostgreSQL database as normal, returns the result to the user, and saves a copy of that result in Redis for a set amount of time (e.g., 5-10 minutes). +* **Your Advantage:** Your Coolify `docker ps` output shows you already have a **Redis container running** as part of Coolify's core services. You can easily create a new, dedicated Redis database for your application within Coolify and start using it. + +You have built an incredibly solid foundation. These optimizations are the natural next steps to take your already impressive project to the next level of performance. Well done! \ No newline at end of file