diff --git a/docs/notes/redis.md b/docs/notes/redis.md new file mode 100644 index 0000000..2bf231e --- /dev/null +++ b/docs/notes/redis.md @@ -0,0 +1,10 @@ +# Implement a Caching Layer with Redis + +For even more speed, especially for data that doesn't change every second, a caching layer is the answer. The goal is to avoid hitting the database at all for repeated requests. + +* **How it Works:** When a request comes in (e.g., for Houston Rockets 2024 stats), your Go application first checks if the result is already in Redis (an extremely fast in-memory database). + * If it's in Redis, it returns the cached data immediately (e.g., in <50ms). + * If it's not in Redis, it queries the PostgreSQL database as normal, returns the result to the user, and saves a copy of that result in Redis for a set amount of time (e.g., 5-10 minutes). +* **Your Advantage:** Your Coolify `docker ps` output shows you already have a **Redis container running** as part of Coolify's core services. You can easily create a new, dedicated Redis database for your application within Coolify and start using it. + +You have built an incredibly solid foundation. These optimizations are the natural next steps to take your already impressive project to the next level of performance. Well done! \ No newline at end of file diff --git a/services/player_shot_chart_scrape_service.go b/services/player_shot_chart_scrape_service.go index 42c973a..cefa62c 100644 --- a/services/player_shot_chart_scrape_service.go +++ b/services/player_shot_chart_scrape_service.go @@ -85,6 +85,9 @@ func FetchAndStoreShotChartScrapedForPlayer( // --- BATCHING LOGIC START --- // Create a slice to hold all the shot data for the current season. var shotsToUpsert []models.PlayerShotChart + // >>>>>>>>>> FIX START: Add a map to track unique shots to prevent duplicates. + uniqueShots := make(map[string]struct{}) + // <<<<<<<<<< FIX END // 4) Scrape every tooltip and collect the data into the slice. wrapper.Find("div.tooltip.make, div.tooltip.miss").Each(func(_ int, s *goquery.Selection) { @@ -130,6 +133,18 @@ func FetchAndStoreShotChartScrapedForPlayer( teamScore, oppScore := mustAtoi(sc[0]), mustAtoi(sc[1]) lead := teamScore > oppScore + // >>>>>>>>>> FIX START: Create a unique key based on the conflict columns. + uniqueKey := fmt.Sprintf("%s|%d|%s|%s|%s|%d|%d", + playerID, season, date, quarter, timeRem, top, left) + + // If we have already seen this key, skip this iteration. + if _, exists := uniqueShots[uniqueKey]; exists { + return + } + uniqueShots[uniqueKey] = struct{}{} + // <<<<<<<<<< FIX END + + shot := models.PlayerShotChart{ PlayerID: playerID, PlayerName: playerName, @@ -156,6 +171,8 @@ func FetchAndStoreShotChartScrapedForPlayer( if len(shotsToUpsert) > 0 { log.Printf("Attempting to batch upsert %d shots for player %s in season %d...", len(shotsToUpsert), playerID, season) + // NOTE: I am assuming the column name for the "Quarter" field is "qtr". + // If not, you must update the clause.OnConflict below. if err := db.Clauses(clause.OnConflict{ Columns: []clause.Column{ // MUST match the unique index order in the model {Name: "player_id"}, {Name: "season"}, {Name: "date"}, @@ -183,6 +200,7 @@ func FetchAndStoreShotChartScrapedForPlayer( // extractCommentedShotChart returns the inner HTML of the comment block that // contains