diff --git a/.DS_Store b/.DS_Store index d977ea7..d5fedcf 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index efb132e..b7b9994 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ data/nba.db /data/* docs/restore_dump_remote.md ./docs/restore_dump_remote.md +docs/notes/conversation_1.md diff --git a/README.md b/README.md index 61854ee..15c8c71 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,251 @@ # NBA_Go -## First‑time bootstrap +A high-performance NBA statistics REST API built with Go (Fiber), PostgreSQL, and NGINX. Data is scraped from Basketball Reference and served through a load-balanced, containerized stack with built-in observability. -```bash -# 1. build + run -docker-compose up --build -d - -# 2. create API key (ADMIN_SECRET is loaded from .env) -curl -XPOST http://localhost:8080/admin/keys \ - -H "X-Admin-Secret: $ADMIN_SECRET" \ - -d '{"label":"local-test"}' -# → { "id":1, "apiKey":"ab12cd…" } - -# 3. call a protected endpoint -curl http://localhost:8080/api/playeradvancedstats \ - -H "X-API-Key: ab12cd…" +## Architecture +``` +┌──────────────────────────────────────────────────────────┐ +│ NGINX (reverse proxy / round-robin load balancer :8080) │ +├──────────────────────────────────────────────────────────┤ +│ API Instance x3 (Fiber :5000 each) │ +│ ┌──────────┐ ┌─────────────┐ ┌───────────────────┐ │ +│ │ Routes │→ │ Controllers │→ │ Services (scraper) │ │ +│ └──────────┘ └─────────────┘ └───────────────────┘ │ +├──────────────────────────────────────────────────────────┤ +│ PostgreSQL 15 (GORM ORM) │ +├──────────────────────────────────────────────────────────┤ +│ Prometheus + Grafana (metrics & dashboards) │ +└──────────────────────────────────────────────────────────┘ ``` -## Swagger Initiate Docs +### Project Structure + +``` +. +├── main.go # Entry point (API server or import-data mode) +├── import.go # Bulk data import orchestration +├── config/ # Database initialization +├── models/ # GORM models (Game, PlayerAdvancedStat, PlayerTotalStat, etc.) +├── controllers/ # HTTP handlers, DTOs, pagination, filtering, sorting +├── routes/ # Route registration grouped by domain +├── services/ # Web scrapers (Basketball Reference via goquery) +├── utils/ +│ ├── middleware/ # Rate limiter, metrics, API key auth +│ ├── metrics/ # Prometheus counter/histogram definitions +│ └── security/ # API key generation & hashing +├── nginx/ # NGINX load balancer config +├── prometheus/ # Prometheus scrape config +├── grafana/ # Pre-provisioned dashboards & datasources +├── docker-compose.yml # Production (Coolify) +├── docker-compose.local.yml # Local development (includes Postgres) +└── docker-compose.override.yml # Override for remote DB development +``` + +## API Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/games` | Game data with box scores, line scores, team/player stats | +| GET | `/api/playeradvancedstats` | Advanced stats (PER, WS, VORP, BPM, etc.) | +| GET | `/api/playertotals` | Season totals (points, rebounds, assists, etc.) | +| GET | `/api/playershotchart` | Shot chart coordinate data | +| GET | `/swagger/*` | Interactive Swagger UI documentation | +| GET | `/metrics` | Prometheus metrics endpoint | +| POST | `/admin/keys` | Create API key (requires `X-Admin-Secret` header) | + +### Query Parameters (all data endpoints) + +| Parameter | Type | Description | +|-----------|------|-------------| +| `page` | int | Page number (default: 1) | +| `pageSize` | int | Results per page (default: 20) | +| `sortBy` | string | Field to sort by (varies per endpoint) | +| `ascending` | bool | Sort direction (default: false / descending) | +| `season` | int | Filter by season year (e.g., 2025) | +| `team` | string | Filter by team abbreviation (e.g., LAL, BOS) | +| `playerId` | string | Filter by player ID (e.g., jamesle01) | +| `isPlayoff` | bool | Filter for playoff stats | + +#### Games-specific parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `date` | string | Filter by date (YYYY-MM-DD) | +| `gameId` | string | Filter by specific game ID | +| `include` | string | Comma-separated associations to preload: `lineScores`, `playerGameBasicStats`, `playerGameAdvStats`, `teamGameBasicStats`, `teamGameAdvStats` | + +### Example Requests + +```bash +# Get top scorers for the 2025 season +curl "http://localhost:8080/api/playertotals?season=2025&sortBy=points&pageSize=10" + +# Get a specific game with full box score +curl "http://localhost:8080/api/games?gameId=202501010LAL&include=lineScores,playerGameBasicStats,teamGameBasicStats" + +# Get LeBron's advanced stats across all seasons +curl "http://localhost:8080/api/playeradvancedstats?playerId=jamesle01&sortBy=season&ascending=true" + +# Get shot chart data for Curry in 2024 +curl "http://localhost:8080/api/playershotchart?playerId=curryst01&season=2024" +``` + +### Response Format + +All endpoints return paginated JSON: + +```json +{ + "data": [...], + "pagination": { + "total": 450, + "page": 1, + "pageSize": 20, + "pages": 23 + } +} +``` + +## Rate Limiting + +The API enforces a per-IP rate limit of **20 requests per minute per instance**. With 3 instances behind NGINX round-robin, the effective limit is ~60 requests/minute per client. + +Exceeding the limit returns: + +```json +HTTP 429 +{"error": "Rate limit exceeded. Try again later."} +``` + +## Getting Started + +### Prerequisites + +- Docker & Docker Compose +- Go 1.23+ (for local development) +- A `.env` file with database credentials + +### Environment Variables + +```env +DB_HOST=postgres +DB_USER=your_user +DB_PASSWORD=your_password +DB_NAME=your_db +DB_PORT=5432 +ADMIN_SECRET=your_admin_secret +``` + +### Local Development + +```bash +# Start everything (Postgres, 3 API instances, NGINX, Prometheus, Grafana) +docker-compose -f docker-compose.local.yml up --build -d + +# Or use the Makefile shortcut +make up +``` + +Services will be available at: + +| Service | URL | +|---------|-----| +| API (via NGINX) | http://localhost:8081 | +| Prometheus | http://localhost:9090 | +| Grafana | http://localhost:3001 (admin/testing) | +| API instance 1 (direct) | http://localhost:5001 | +| API instance 2 (direct) | http://localhost:5002 | +| API instance 3 (direct) | http://localhost:5003 | + +### Importing Data + +The application has a dual-mode entry point. To run the initial data import (migrations + scraping): + +```bash +docker-compose -f docker-compose.local.yml run --rm db-init +``` + +This runs `main.go` with the `import-data` argument, which: +1. Runs all GORM AutoMigrate operations +2. Scrapes Basketball Reference for player advanced stats, totals, game schedules, and box scores +3. Upserts all data into PostgreSQL + +### Stopping + +```bash +docker compose down +# or +make down +``` + +## Production Deployment + +The main `docker-compose.yml` is configured for deployment on Coolify with an external `coolify` network. It expects the database to be provisioned separately (no local Postgres service). + +The `docker-compose.override.yml` disables the local Postgres container and removes `depends_on` constraints, allowing API services to connect to a remote database specified in `.env`. + +## Observability + +### Prometheus Metrics + +Exposed at `/metrics` on each API instance. Tracked metrics: + +- `nba_http_requests_total` — counter by method, endpoint, status +- `nba_http_request_duration_seconds` — histogram by method, endpoint +- `nba_db_operations_total` — counter by operation, entity + +### Grafana + +Pre-provisioned dashboards visualize request rates and endpoint usage. Access at port 3001 (local) or 3000 (production). + +## API Key Management (Optional) + +API key authentication is available but currently disabled. To create keys for future use: + +```bash +# Create a key +curl -XPOST http://localhost:8080/admin/keys \ + -H "X-Admin-Secret: $ADMIN_SECRET" \ + -d '{"label":"my-app"}' +# → {"id":1, "apiKey":"ab12cd…"} + +# Revoke a key +curl -XPOST http://localhost:8080/admin/keys/1/revoke \ + -H "X-Admin-Secret: $ADMIN_SECRET" +``` + +To enforce API keys, uncomment `app.Use(middleware.APIKeyAuth(db))` in `main.go`. + +## Regenerating Swagger Docs ```bash swag init -g main.go -o docs ``` -## Test +## Running Tests ```bash -go run loadtest.go -n 100 -c 10 -url "http://127.0.0.1:8080/api/playeradvancedstats?page=1&pageSize=20" -log results.log -key "xxx" +go test -v . ``` -## Local Environment +### Load Testing ```bash -docker compose down +cd test +go run loadtest.go -n 100 -c 10 -url "http://localhost:8080/api/playeradvancedstats?page=1&pageSize=20" -log results.log ``` -```bash -docker-compose -f docker-compose.local.yml up --build -d -``` +## Tech Stack + +| Component | Technology | +|-----------|-----------| +| Language | Go 1.23+ | +| Framework | Fiber v2 | +| ORM | GORM | +| Database | PostgreSQL 15 | +| Scraping | goquery | +| Load Balancer | NGINX | +| Monitoring | Prometheus + Grafana | +| Docs | Swagger (swaggo) | +| Containerization | Docker + Docker Compose | diff --git a/docker-compose.local.yml b/docker-compose.local.yml index fafea45..0755bf8 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -75,6 +75,7 @@ services: image: 'nginx:stable' volumes: - './nginx/nginx.conf:/etc/nginx/nginx.conf:ro' + - nginx_cache:/var/cache/nginx depends_on: - api1 - api2 @@ -123,6 +124,7 @@ volumes: postgres_data: # New volume for Postgres prometheus_data: grafana_data: + nginx_cache: networks: api.network: null diff --git a/docker-compose.yml b/docker-compose.yml index 4b1fb55..dc14cca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: image: 'nginx:stable' volumes: - '/data/nba-go-config/nginx:/etc/nginx:ro' + - nginx_cache:/var/cache/nginx depends_on: - api1 - api2 @@ -84,6 +85,7 @@ services: volumes: prometheus_data: grafana_data: + nginx_cache: networks: # api_network: diff --git a/go.mod b/go.mod index 189538f..e3c92c4 100644 --- a/go.mod +++ b/go.mod @@ -45,12 +45,14 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect + github.com/tinylib/msgp v1.2.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect diff --git a/go.sum b/go.sum index 81c09c6..aee29bd 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= @@ -121,6 +123,8 @@ github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9J github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ= github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= +github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= diff --git a/import.go b/import.go index 5c75c86..2c12b7f 100644 --- a/import.go +++ b/import.go @@ -17,7 +17,7 @@ func importPlayerAdvanced(db *gorm.DB) { log.Printf("advanced import failed for %d: %v", season, err) } log.Printf("Advanced import for season: %d", season) - time.Sleep(1300 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) utils.SleepWithJitter(1000 * time.Millisecond) } } @@ -29,7 +29,7 @@ func importPlayerAdvancedPlayoffs(db *gorm.DB) { log.Printf("advanced import failed for %d: %v", season, err) } log.Printf("Advanced Playoffs import for season: %d", season) - time.Sleep(1300 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) utils.SleepWithJitter(1500 * time.Millisecond) } } @@ -41,7 +41,7 @@ func importPlayerTotalsScrape(db *gorm.DB) { log.Printf("scraped totals import failed for %d: %v", season, err) } log.Printf("Player Totals import for season: %d", season) - time.Sleep(1300 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) utils.SleepWithJitter(1250 * time.Millisecond) } } @@ -53,7 +53,7 @@ func importPlayerTotalsPlayoffsScrape(db *gorm.DB) { log.Printf("scraped playoffs import failed for %d: %v", season, err) } log.Printf("Player Playoffs Totals import for season: %d", season) - time.Sleep(1300 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) utils.SleepWithJitter(1700 * time.Millisecond) } } @@ -64,7 +64,7 @@ func importGameSchedules(db *gorm.DB) { months := []string{ // "september", "october", "november", "december", "january", // "february", "march", "april", "may", "june", - "february", "march", "april", "may", "june", + "may", "june", // "february", "march", "april", "may", "june", // "october", "november", "december", } @@ -79,7 +79,7 @@ func importGameSchedules(db *gorm.DB) { } log.Printf("Game schedule import for %s, %d complete.", month, season) // Respectful delay between requests - time.Sleep(1400 * time.Millisecond) + time.Sleep(2500 * time.Millisecond) utils.SleepWithJitter(1800 * time.Millisecond) } log.Printf("--- Finished Game Schedule Import for Season: %d ---", season) @@ -91,9 +91,9 @@ func importGameSchedules(db *gorm.DB) { func importBoxScores(db *gorm.DB) { // now := time.Now() - from := time.Date(2026, time.April, 10, 0, 0, 0, 0, time.UTC) + from := time.Date(2026, time.April, 28, 0, 0, 0, 0, time.UTC) // from := time.Date(now.Year(), now.Month(), now.Day()-1, 5, 30, 0, 0, time.UTC) - to := time.Date(2026, time.May, 1, 5, 30, 0, 0, time.UTC) + to := time.Date(2026, time.June, 15, 5, 30, 0, 0, time.UTC) dateRangeComment := fmt.Sprintf("--- Starting Box Score Data Import for games between %s and %s ---", from.Format("January 2, 2006"), diff --git a/main.go b/main.go index 05e7143..20365e7 100644 --- a/main.go +++ b/main.go @@ -96,6 +96,7 @@ func main() { // middlewares app.Use(logger.New()) + app.Use(middleware.RateLimiter()) app.Use(middleware.MetricsMiddleware()) // DB connection (no migrations on API startup) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c76f90f..134b2b7 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -5,6 +5,8 @@ events { } http { + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=api_cache:10m max_size=100m inactive=5m; + upstream go_backend { server api1:5000; server api2:5000; @@ -17,6 +19,24 @@ http { access_log /dev/stdout; error_log /dev/stderr; + location /api/ { + proxy_cache api_cache; + proxy_cache_valid 200 30s; + proxy_cache_key "$request_uri"; + proxy_cache_use_stale error timeout updating; + add_header X-Cache-Status $upstream_cache_status; + + proxy_pass http://go_backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location / { proxy_pass http://go_backend; proxy_http_version 1.1; @@ -29,4 +49,4 @@ http { proxy_set_header X-Forwarded-Proto $scheme; } } -} \ No newline at end of file +} diff --git a/utils/middleware/rate_limiter.go b/utils/middleware/rate_limiter.go new file mode 100644 index 0000000..8ec30e8 --- /dev/null +++ b/utils/middleware/rate_limiter.go @@ -0,0 +1,36 @@ +package middleware + +import ( + "strings" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/limiter" +) + +func RateLimiter() fiber.Handler { + return limiter.New(limiter.Config{ + Max: 20, + Expiration: 1 * time.Minute, + Next: func(c *fiber.Ctx) bool { + // Skip rate limiting for internal services and infra endpoints + ip := c.IP() + if strings.HasPrefix(ip, "10.") || strings.HasPrefix(ip, "172.") || ip == "127.0.0.1" { + return true + } + path := c.Path() + return path == "/metrics" || strings.HasPrefix(path, "/swagger") + }, + KeyGenerator: func(c *fiber.Ctx) string { + if ip := c.Get("X-Real-IP"); ip != "" { + return ip + } + return c.IP() + }, + LimitReached: func(c *fiber.Ctx) error { + return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{ + "error": "Rate limit exceeded. Try again later.", + }) + }, + }) +}