feat(database): add PostgreSQL support with native pgx pool

Add PostgreSQL support to database package alongside existing MySQL support. Both databases share common infrastructure (pool management, metrics, transactions) while using database-specific connectors. database/ changes: - Add PostgresConfig struct and PostgreSQL connector using pgx/stdlib - Change MySQL config from DBConfig to *MySQLConfig (pointer) - Add Config.Validate() to prevent multiple database configs - Add PostgreSQL connector with secure config building (no password in DSN) - Add field validation and secure defaults (SSLMode="prefer") - Support legacy flat PostgreSQL config format for backward compatibility - Add tests for PostgreSQL configs and validation New database/pgdb/ package: - Native pgx connection pool support (*pgxpool.Pool) - OpenPool() and OpenPoolWithConfigFile() APIs - CreatePoolConfig() for secure config conversion - PoolOptions for fine-grained pool control - Full test coverage and documentation Security: - Passwords never exposed in DSN strings - Set passwords separately in pgx config objects - Validate all configuration before connection Architecture: - Shared code in database/ for both MySQL and PostgreSQL (sql.DB) - database/pgdb/ for PostgreSQL-specific native pool support
2025-09-27 16:55:54 -07:00
parent 4767caf7b8
commit 45308cd4bf
9 changed files with 832 additions and 43 deletions
--- a/database/pgdb/CLAUDE.md
+++ b/database/pgdb/CLAUDE.md
@@ -0,0 +1,120 @@
+# pgdb - Native PostgreSQL Connection Pool
+
+Primary package for PostgreSQL connections using native pgx pool (`*pgxpool.Pool`). Provides better performance and PostgreSQL-specific features compared to `database/sql`.
+
+## Usage
+
+### Basic Example
+
+```go
+import (
+    "context"
+    "go.ntppool.org/common/database/pgdb"
+)
+
+func main() {
+    ctx := context.Background()
+
+    // Open pool with default options
+    pool, err := pgdb.OpenPool(ctx, pgdb.DefaultPoolOptions())
+    if err != nil {
+        log.Fatal(err)
+    }
+    defer pool.Close()
+
+    // Use the pool for queries
+    row := pool.QueryRow(ctx, "SELECT version()")
+    var version string
+    row.Scan(&version)
+}
+```
+
+### With Custom Config File
+
+```go
+pool, err := pgdb.OpenPoolWithConfigFile(ctx, "/path/to/database.yaml")
+```
+
+### With Custom Pool Settings
+
+```go
+opts := pgdb.DefaultPoolOptions()
+opts.MaxConns = 50
+opts.MinConns = 5
+opts.MaxConnLifetime = 2 * time.Hour
+
+pool, err := pgdb.OpenPool(ctx, opts)
+```
+
+## Configuration Format
+
+### Recommended: Nested Format (database.yaml)
+
+```yaml
+postgres:
+  host: localhost
+  port: 5432
+  user: myuser
+  pass: mypassword
+  name: mydb
+  sslmode: prefer
+```
+
+### Legacy: Flat Format (backward compatible)
+
+```yaml
+host: localhost
+port: 5432
+user: myuser
+pass: mypassword
+name: mydb
+sslmode: prefer
+```
+
+## Configuration Options
+
+### PoolOptions
+
+- `ConfigFiles` - List of config file paths to search (default: `database.yaml`, `/vault/secrets/database.yaml`)
+- `MinConns` - Minimum connections (default: 0)
+- `MaxConns` - Maximum connections (default: 25)
+- `MaxConnLifetime` - Connection lifetime (default: 1 hour)
+- `MaxConnIdleTime` - Idle timeout (default: 30 minutes)
+- `HealthCheckPeriod` - Health check interval (default: 1 minute)
+
+### PostgreSQL Config Fields
+
+- `host` - Database host (required)
+- `user` - Database user (required)
+- `pass` - Database password
+- `name` - Database name (required)
+- `port` - Port number (default: 5432)
+- `sslmode` - SSL mode: `disable`, `allow`, `prefer`, `require`, `verify-ca`, `verify-full` (default: `prefer`)
+
+## Environment Variables
+
+- `DATABASE_CONFIG_FILE` - Override config file location
+
+## When to Use
+
+**Use `pgdb.OpenPool()`** (this package) when:
+- You need native PostgreSQL features (LISTEN/NOTIFY, COPY, etc.)
+- You want better performance
+- You're writing new PostgreSQL code
+
+**Use `database.OpenDB()`** (sql.DB) when:
+- You need database-agnostic code
+- You're using SQLC or other tools that expect `database/sql`
+- You need to support both MySQL and PostgreSQL
+
+## Security
+
+This package avoids password exposure by:
+1. Never constructing DSN strings with passwords
+2. Setting passwords separately in pgx config objects
+3. Validating all configuration before connection
+
+## See Also
+
+- `database/` - Generic sql.DB support for MySQL and PostgreSQL
+- pgx documentation: https://github.com/jackc/pgx
--- a/database/pgdb/config.go
+++ b/database/pgdb/config.go
@@ -0,0 +1,64 @@
+package pgdb
+
+import (
+	"fmt"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.ntppool.org/common/database"
+)
+
+// CreatePoolConfig converts database.PostgresConfig to pgxpool.Config
+// This is the secure way to create a config without exposing passwords in DSN strings
+func CreatePoolConfig(cfg *database.PostgresConfig) (*pgxpool.Config, error) {
+	// Validate required fields
+	if cfg.Host == "" {
+		return nil, fmt.Errorf("postgres: host is required")
+	}
+	if cfg.User == "" {
+		return nil, fmt.Errorf("postgres: user is required")
+	}
+	if cfg.Name == "" {
+		return nil, fmt.Errorf("postgres: database name is required")
+	}
+
+	// Validate SSLMode
+	validSSLModes := map[string]bool{
+		"disable": true, "allow": true, "prefer": true,
+		"require": true, "verify-ca": true, "verify-full": true,
+	}
+	if cfg.SSLMode != "" && !validSSLModes[cfg.SSLMode] {
+		return nil, fmt.Errorf("postgres: invalid sslmode: %s", cfg.SSLMode)
+	}
+
+	// Set defaults
+	host := cfg.Host
+	if host == "" {
+		host = "localhost"
+	}
+
+	port := cfg.Port
+	if port == 0 {
+		port = 5432
+	}
+
+	sslmode := cfg.SSLMode
+	if sslmode == "" {
+		sslmode = "prefer"
+	}
+
+	// Build connection string WITHOUT password
+	// We'll set the password separately in the config
+	connString := fmt.Sprintf("host=%s port=%d user=%s dbname=%s sslmode=%s",
+		host, port, cfg.User, cfg.Name, sslmode)
+
+	// Parse the connection string
+	poolConfig, err := pgxpool.ParseConfig(connString)
+	if err != nil {
+		return nil, fmt.Errorf("postgres: failed to parse connection config: %w", err)
+	}
+
+	// Set password separately (security: never put password in the connection string)
+	poolConfig.ConnConfig.Password = cfg.Pass
+
+	return poolConfig, nil
+}
--- a/database/pgdb/pool.go
+++ b/database/pgdb/pool.go
@@ -0,0 +1,173 @@
+package pgdb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.ntppool.org/common/database"
+	"gopkg.in/yaml.v3"
+)
+
+// PoolOptions configures pgxpool connection behavior
+type PoolOptions struct {
+	// ConfigFiles is a list of config file paths to search for database configuration
+	ConfigFiles []string
+
+	// MinConns is the minimum number of connections in the pool
+	// Default: 0 (no minimum)
+	MinConns int32
+
+	// MaxConns is the maximum number of connections in the pool
+	// Default: 25
+	MaxConns int32
+
+	// MaxConnLifetime is the maximum lifetime of a connection
+	// Default: 1 hour
+	MaxConnLifetime time.Duration
+
+	// MaxConnIdleTime is the maximum idle time of a connection
+	// Default: 30 minutes
+	MaxConnIdleTime time.Duration
+
+	// HealthCheckPeriod is how often to check connection health
+	// Default: 1 minute
+	HealthCheckPeriod time.Duration
+}
+
+// DefaultPoolOptions returns sensible defaults for pgxpool
+func DefaultPoolOptions() PoolOptions {
+	return PoolOptions{
+		ConfigFiles:       getConfigFiles(),
+		MinConns:          0,
+		MaxConns:          25,
+		MaxConnLifetime:   time.Hour,
+		MaxConnIdleTime:   30 * time.Minute,
+		HealthCheckPeriod: time.Minute,
+	}
+}
+
+// OpenPool opens a native pgx connection pool with the specified configuration
+// This is the primary and recommended way to connect to PostgreSQL
+func OpenPool(ctx context.Context, options PoolOptions) (*pgxpool.Pool, error) {
+	// Find and read config file
+	pgCfg, err := findAndParseConfig(options.ConfigFiles)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create pool config from PostgreSQL config
+	poolConfig, err := CreatePoolConfig(pgCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Apply pool-specific settings
+	poolConfig.MinConns = options.MinConns
+	poolConfig.MaxConns = options.MaxConns
+	poolConfig.MaxConnLifetime = options.MaxConnLifetime
+	poolConfig.MaxConnIdleTime = options.MaxConnIdleTime
+	poolConfig.HealthCheckPeriod = options.HealthCheckPeriod
+
+	// Create the pool
+	pool, err := pgxpool.NewWithConfig(ctx, poolConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create connection pool: %w", err)
+	}
+
+	// Test the connection
+	if err := pool.Ping(ctx); err != nil {
+		pool.Close()
+		return nil, fmt.Errorf("failed to ping database: %w", err)
+	}
+
+	return pool, nil
+}
+
+// OpenPoolWithConfigFile opens a connection pool using an explicit config file path
+// This is a convenience function for when you have a specific config file
+func OpenPoolWithConfigFile(ctx context.Context, configFile string) (*pgxpool.Pool, error) {
+	options := DefaultPoolOptions()
+	options.ConfigFiles = []string{configFile}
+	return OpenPool(ctx, options)
+}
+
+// findAndParseConfig searches for and parses the first existing config file
+func findAndParseConfig(configFiles []string) (*database.PostgresConfig, error) {
+	var firstErr error
+
+	for _, configFile := range configFiles {
+		if configFile == "" {
+			continue
+		}
+
+		// Check if file exists
+		if _, err := os.Stat(configFile); err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+			continue
+		}
+
+		// Try to read and parse the file
+		pgCfg, err := parseConfigFile(configFile)
+		if err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+			continue
+		}
+
+		return pgCfg, nil
+	}
+
+	if firstErr != nil {
+		return nil, fmt.Errorf("no config file found: %w", firstErr)
+	}
+	return nil, fmt.Errorf("no valid config files provided")
+}
+
+// parseConfigFile reads and parses a YAML config file
+func parseConfigFile(configFile string) (*database.PostgresConfig, error) {
+	file, err := os.Open(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open config file: %w", err)
+	}
+	defer file.Close()
+
+	dec := yaml.NewDecoder(file)
+	cfg := database.Config{}
+
+	if err := dec.Decode(&cfg); err != nil {
+		return nil, fmt.Errorf("failed to decode config: %w", err)
+	}
+
+	// Extract PostgreSQL config
+	if cfg.Postgres != nil {
+		return cfg.Postgres, nil
+	}
+
+	// Check for legacy flat format
+	if cfg.User != "" && cfg.Name != "" {
+		return &database.PostgresConfig{
+			User:    cfg.User,
+			Pass:    cfg.Pass,
+			Host:    cfg.Host,
+			Port:    cfg.Port,
+			Name:    cfg.Name,
+			SSLMode: cfg.SSLMode,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("no PostgreSQL configuration found in %s", configFile)
+}
+
+// getConfigFiles returns the list of config files to search
+func getConfigFiles() []string {
+	if configFile := os.Getenv("DATABASE_CONFIG_FILE"); configFile != "" {
+		return []string{configFile}
+	}
+	return []string{"database.yaml", "/vault/secrets/database.yaml"}
+}
--- a/database/pgdb/pool_test.go
+++ b/database/pgdb/pool_test.go
@@ -0,0 +1,151 @@
+package pgdb
+
+import (
+	"testing"
+	"time"
+
+	"go.ntppool.org/common/database"
+)
+
+func TestCreatePoolConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *database.PostgresConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			cfg: &database.PostgresConfig{
+				Host:    "localhost",
+				Port:    5432,
+				User:    "testuser",
+				Pass:    "testpass",
+				Name:    "testdb",
+				SSLMode: "require",
+			},
+			wantErr: false,
+		},
+		{
+			name: "valid config with defaults",
+			cfg: &database.PostgresConfig{
+				Host: "localhost",
+				User: "testuser",
+				Pass: "testpass",
+				Name: "testdb",
+				// Port and SSLMode will use defaults
+			},
+			wantErr: false,
+		},
+		{
+			name: "missing host",
+			cfg: &database.PostgresConfig{
+				User: "testuser",
+				Pass: "testpass",
+				Name: "testdb",
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing user",
+			cfg: &database.PostgresConfig{
+				Host: "localhost",
+				Pass: "testpass",
+				Name: "testdb",
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing database name",
+			cfg: &database.PostgresConfig{
+				Host: "localhost",
+				User: "testuser",
+				Pass: "testpass",
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid sslmode",
+			cfg: &database.PostgresConfig{
+				Host:    "localhost",
+				User:    "testuser",
+				Pass:    "testpass",
+				Name:    "testdb",
+				SSLMode: "invalid",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			poolCfg, err := CreatePoolConfig(tt.cfg)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CreatePoolConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !tt.wantErr && poolCfg == nil {
+				t.Error("CreatePoolConfig() returned nil config without error")
+			}
+			if !tt.wantErr && poolCfg != nil {
+				// Verify config fields are set correctly
+				if poolCfg.ConnConfig.Host != tt.cfg.Host && tt.cfg.Host != "" {
+					t.Errorf("Expected Host=%s, got %s", tt.cfg.Host, poolCfg.ConnConfig.Host)
+				}
+				if poolCfg.ConnConfig.User != tt.cfg.User {
+					t.Errorf("Expected User=%s, got %s", tt.cfg.User, poolCfg.ConnConfig.User)
+				}
+				if poolCfg.ConnConfig.Password != tt.cfg.Pass {
+					t.Errorf("Expected Password to be set correctly")
+				}
+				if poolCfg.ConnConfig.Database != tt.cfg.Name {
+					t.Errorf("Expected Database=%s, got %s", tt.cfg.Name, poolCfg.ConnConfig.Database)
+				}
+			}
+		})
+	}
+}
+
+func TestDefaultPoolOptions(t *testing.T) {
+	opts := DefaultPoolOptions()
+
+	// Verify expected defaults
+	if opts.MinConns != 0 {
+		t.Errorf("Expected MinConns=0, got %d", opts.MinConns)
+	}
+	if opts.MaxConns != 25 {
+		t.Errorf("Expected MaxConns=25, got %d", opts.MaxConns)
+	}
+	if opts.MaxConnLifetime != time.Hour {
+		t.Errorf("Expected MaxConnLifetime=1h, got %v", opts.MaxConnLifetime)
+	}
+	if opts.MaxConnIdleTime != 30*time.Minute {
+		t.Errorf("Expected MaxConnIdleTime=30m, got %v", opts.MaxConnIdleTime)
+	}
+	if opts.HealthCheckPeriod != time.Minute {
+		t.Errorf("Expected HealthCheckPeriod=1m, got %v", opts.HealthCheckPeriod)
+	}
+	if len(opts.ConfigFiles) == 0 {
+		t.Error("Expected ConfigFiles to be non-empty")
+	}
+}
+
+func TestCreatePoolConfigDefaults(t *testing.T) {
+	// Test that defaults are applied correctly
+	cfg := &database.PostgresConfig{
+		Host: "localhost",
+		User: "testuser",
+		Pass: "testpass",
+		Name: "testdb",
+		// Port and SSLMode not set
+	}
+
+	poolCfg, err := CreatePoolConfig(cfg)
+	if err != nil {
+		t.Fatalf("CreatePoolConfig() failed: %v", err)
+	}
+
+	// Verify defaults were applied
+	if poolCfg.ConnConfig.Port != 5432 {
+		t.Errorf("Expected default Port=5432, got %d", poolCfg.ConnConfig.Port)
+	}
+}