feat(db): migrate from MySQL to PostgreSQL

Replace MySQL driver with pgx/v5 and pgxpool:
- Update sqlc to use postgresql engine
- Convert query.sql to PostgreSQL syntax ($1 params, CASE WHEN,
  ANY() arrays)
- Replace sql.DB with pgxpool.Pool throughout
- Change nullable types from sql.Null* to pgtype.*
- Update ID types from uint32 to int64 for PostgreSQL compatibility
- Delete MySQL-specific dynamic_connect.go
- Add opentelemetry.gowrap template for tracing

server/dnsanswers.go

@@ -1,11 +1,11 @@
 package server
 
 import (
-	"database/sql"
 	"errors"
 	"net/http"
 	"net/netip"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/labstack/echo/v4"
 	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/sync/errgroup"
@@ -56,7 +56,7 @@ func (srv *Server) dnsAnswers(c echo.Context) error {
 	queryGroup, ctx := errgroup.WithContext(ctx)
 
 	var zoneStats []ntpdb.GetZoneStatsV2Row
-	var serverNetspeed uint32
+	var serverNetspeed int64
 
 	queryGroup.Go(func() error {
 		var err error
@@ -64,7 +64,7 @@ func (srv *Server) dnsAnswers(c echo.Context) error {
 
 		serverNetspeed, err = q.GetServerNetspeed(ctx, ip.String())
 		if err != nil {
-			if !errors.Is(err, sql.ErrNoRows) {
+			if !errors.Is(err, pgx.ErrNoRows) {
 				log.Error("GetServerNetspeed", "err", err)
 			}
 			return err // this will return if the server doesn't exist
@@ -116,7 +116,7 @@ func (srv *Server) dnsAnswers(c echo.Context) error {
 
 	err = queryGroup.Wait()
 	if err != nil {
-		if errors.Is(err, sql.ErrNoRows) {
+		if errors.Is(err, pgx.ErrNoRows) {
 			return c.String(http.StatusNotFound, "Not found")
 		}
 		log.Error("query error", "err", err)
@@ -130,7 +130,7 @@ func (srv *Server) dnsAnswers(c echo.Context) error {
 		if zn == "@" {
 			zn = ""
 		}
-		zoneTotals[zn] = z.NetspeedActive // binary.BigEndian.Uint64(...)
+		zoneTotals[zn] = int(z.NetspeedActive) // binary.BigEndian.Uint64(...)
 		// log.Info("zone netspeed", "cc", z.ZoneName, "speed", z.NetspeedActive)
 	}
 

server/functions.go

@@ -2,12 +2,12 @@ package server
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"net/netip"
 	"strconv"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"go.ntppool.org/common/logger"
 	"go.ntppool.org/common/tracing"
 	"go.ntppool.org/data-api/ntpdb"
@@ -22,7 +22,7 @@ func (srv *Server) FindServer(ctx context.Context, serverID string) (ntpdb.Serve
 	var serverData ntpdb.Server
 	var dberr error
 	if id, err := strconv.Atoi(serverID); id > 0 && err == nil {
-		serverData, dberr = q.GetServerByID(ctx, uint32(id))
+		serverData, dberr = q.GetServerByID(ctx, int64(id))
 	} else {
 		ip, err := netip.ParseAddr(serverID)
 		if err != nil || !ip.IsValid() {
@@ -31,7 +31,7 @@ func (srv *Server) FindServer(ctx context.Context, serverID string) (ntpdb.Serve
 		serverData, dberr = q.GetServerByIP(ctx, ip.String())
 	}
 	if dberr != nil {
-		if !errors.Is(dberr, sql.ErrNoRows) {
+		if !errors.Is(dberr, pgx.ErrNoRows) {
 			log.Error("could not query server id", "err", dberr)
 			return serverData, dberr
 		}

server/grafana.go

@@ -195,7 +195,7 @@ func transformToGrafanaTableFormat(history *logscores.LogScoreHistory, monitors
 			skippedInvalidMonitors++
 			continue
 		}
-		monitorID := int(ls.MonitorID.Int32)
+		monitorID := int(ls.MonitorID.Int64)
 		monitorData[monitorID] = append(monitorData[monitorID], ls)
 	}
 
@@ -275,7 +275,7 @@ func transformToGrafanaTableFormat(history *logscores.LogScoreHistory, monitors
 		var values [][]interface{}
 		for _, ls := range logScores {
 			// Convert timestamp to milliseconds
-			timestampMs := ls.Ts.Unix() * 1000
+			timestampMs := ls.Ts.Time.Unix() * 1000
 
 			// Create row: [time, score, rtt, offset]
 			row := []interface{}{
@@ -382,7 +382,7 @@ func (srv *Server) scoresTimeRange(c echo.Context) error {
 		"time_range_duration", params.to.Sub(params.from).String(),
 	)
 
-	logScores, err := srv.ch.LogscoresTimeRange(ctx, int(server.ID), params.monitorID, params.from, params.to, params.maxDataPoints)
+	logScores, err := srv.ch.LogscoresTimeRange(ctx, int(server.ID), int(params.monitorID), params.from, params.to, params.maxDataPoints)
 	if err != nil {
 		log.ErrorContext(ctx, "clickhouse time range query", "err", err,
 			"server_id", server.ID,
@@ -397,8 +397,8 @@ func (srv *Server) scoresTimeRange(c echo.Context) error {
 	log.InfoContext(ctx, "clickhouse query results",
 		"server_id", server.ID,
 		"rows_returned", len(logScores),
-		"first_few_ids", func() []uint64 {
-			ids := make([]uint64, 0, 3)
+		"first_few_ids", func() []int64 {
+			ids := make([]int64, 0, 3)
 			for i, ls := range logScores {
 				if i >= 3 {
 					break
@@ -416,10 +416,10 @@ func (srv *Server) scoresTimeRange(c echo.Context) error {
 	}
 
 	// Get monitor names for the returned data
-	monitorIDs := []uint32{}
+	monitorIDs := []int64{}
 	for _, ls := range logScores {
 		if ls.MonitorID.Valid {
-			monitorID := uint32(ls.MonitorID.Int32)
+			monitorID := ls.MonitorID.Int64
 			if _, exists := history.Monitors[int(monitorID)]; !exists {
 				history.Monitors[int(monitorID)] = ""
 				monitorIDs = append(monitorIDs, monitorID)

server/history.go

@@ -3,7 +3,6 @@ package server
 import (
 	"bytes"
 	"context"
-	"database/sql"
 	"encoding/csv"
 	"errors"
 	"fmt"
@@ -15,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/labstack/echo/v4"
 	"go.ntppool.org/common/logger"
 	"go.ntppool.org/common/tracing"
@@ -63,7 +64,7 @@ func paramHistoryMode(s string) historyMode {
 
 type historyParameters struct {
 	limit       int
-	monitorID   int
+	monitorID   int64
 	server      ntpdb.Server
 	since       time.Time
 	fullHistory bool
@@ -90,7 +91,7 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 
 	monitorParam := c.QueryParam("monitor")
 
-	var monitorID uint32
+	var monitorID int64
 	switch monitorParam {
 	case "":
 		name := "recentmedian.scores.ntp.dev"
@@ -101,7 +102,7 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 			ipVersion = ntpdb.NullMonitorsIpVersion{MonitorsIpVersion: ntpdb.MonitorsIpVersionV6, Valid: true}
 		}
 		monitor, err := q.GetMonitorByNameAndIPVersion(ctx, ntpdb.GetMonitorByNameAndIPVersionParams{
-			TlsName:   sql.NullString{Valid: true, String: name},
+			TlsName:   pgtype.Text{Valid: true, String: name},
 			IpVersion: ipVersion,
 		})
 		if err != nil {
@@ -111,9 +112,9 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 	case "*":
 		monitorID = 0 // don't filter on monitor ID
 	default:
-		mID, err := strconv.ParseUint(monitorParam, 10, 32)
+		mID, err := strconv.ParseInt(monitorParam, 10, 64)
 		if err == nil {
-			monitorID = uint32(mID)
+			monitorID = mID
 		} else {
 			// only accept the name prefix; no wildcards; trust the database
 			// to filter out any other crazy
@@ -129,11 +130,11 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 				ipVersion = ntpdb.NullMonitorsIpVersion{MonitorsIpVersion: ntpdb.MonitorsIpVersionV6, Valid: true}
 			}
 			monitor, err := q.GetMonitorByNameAndIPVersion(ctx, ntpdb.GetMonitorByNameAndIPVersionParams{
-				TlsName:   sql.NullString{Valid: true, String: monitorParam},
+				TlsName:   pgtype.Text{Valid: true, String: monitorParam},
 				IpVersion: ipVersion,
 			})
 			if err != nil {
-				if err == sql.ErrNoRows {
+				if errors.Is(err, pgx.ErrNoRows) {
 					return p, echo.NewHTTPError(http.StatusNotFound, "monitor not found").WithInternal(err)
 				}
 				log.WarnContext(ctx, "could not find monitor", "name", monitorParam, "ip_version", server.IpVersion, "err", err)
@@ -144,7 +145,7 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 		}
 	}
 
-	p.monitorID = int(monitorID)
+	p.monitorID = monitorID
 	log.DebugContext(ctx, "monitor param", "monitor", monitorID, "ip_version", server.IpVersion)
 
 	since, _ := strconv.ParseInt(c.QueryParam("since"), 10, 64) // defaults to 0 so don't care if it parses
@@ -170,8 +171,8 @@ func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context, ser
 	return p, nil
 }
 
-func (srv *Server) getHistoryMySQL(ctx context.Context, _ echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) {
-	ls, err := logscores.GetHistoryMySQL(ctx, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit)
+func (srv *Server) getHistoryPostgres(ctx context.Context, _ echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) {
+	ls, err := logscores.GetHistoryPostgres(ctx, srv.db, p.server.ID, p.monitorID, p.since, p.limit)
 	return ls, err
 }
 
@@ -230,9 +231,9 @@ func (srv *Server) history(c echo.Context) error {
 	}
 
 	if sourceParam == "m" {
-		history, err = srv.getHistoryMySQL(ctx, c, p)
+		history, err = srv.getHistoryPostgres(ctx, c, p)
 	} else {
-		history, err = logscores.GetHistoryClickHouse(ctx, srv.ch, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit, p.fullHistory)
+		history, err = logscores.GetHistoryClickHouse(ctx, srv.ch, srv.db, p.server.ID, p.monitorID, p.since, p.limit, p.fullHistory)
 	}
 	if err != nil {
 		var httpError *echo.HTTPError
@@ -276,7 +277,7 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb
 	}
 
 	type MonitorEntry struct {
-		ID     uint32   `json:"id"`
+		ID     int64    `json:"id"`
 		Name   string   `json:"name"`
 		Type   string   `json:"type"`
 		Ts     string   `json:"ts"`
@@ -297,9 +298,9 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb
 
 	// log.InfoContext(ctx, "monitor id list", "ids", history.MonitorIDs)
 
-	monitorIDs := []uint32{}
+	monitorIDs := []int64{}
 	for k := range history.Monitors {
-		monitorIDs = append(monitorIDs, uint32(k))
+		monitorIDs = append(monitorIDs, int64(k))
 	}
 
 	q := ntpdb.NewWrappedQuerier(ntpdb.New(srv.db))
@@ -318,12 +319,12 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb
 	// log.InfoContext(ctx, "got logScoreMonitors", "count", len(logScoreMonitors))
 
 	// Calculate average RTT per monitor
-	monitorRttSums := make(map[uint32]float64)
-	monitorRttCounts := make(map[uint32]int)
+	monitorRttSums := make(map[int64]float64)
+	monitorRttCounts := make(map[int64]int)
 
 	for _, ls := range history.LogScores {
 		if ls.MonitorID.Valid && ls.Rtt.Valid {
-			monitorID := uint32(ls.MonitorID.Int32)
+			monitorID := ls.MonitorID.Int64
 			monitorRttSums[monitorID] += float64(ls.Rtt.Int32) / 1000.0
 			monitorRttCounts[monitorID]++
 		}
@@ -362,8 +363,8 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb
 		x := float64(1000000000000)
 		score := math.Round(ls.Score*x) / x
 		res.History[i] = ScoresEntry{
-			TS:        ls.Ts.Unix(),
-			MonitorID: int(ls.MonitorID.Int32),
+			TS:        ls.Ts.Time.Unix(),
+			MonitorID: int(ls.MonitorID.Int64),
 			Step:      ls.Step,
 			Score:     score,
 		}
@@ -414,7 +415,7 @@ func (srv *Server) historyCSV(ctx context.Context, c echo.Context, history *logs
 		score := ff(l.Score)
 		var monName string
 		if l.MonitorID.Valid {
-			monName = history.Monitors[int(l.MonitorID.Int32)]
+			monName = history.Monitors[int(l.MonitorID.Int64)]
 		}
 		var leap string
 		if l.Attributes.Leap != 0 {
@@ -427,13 +428,13 @@ func (srv *Server) historyCSV(ctx context.Context, c echo.Context, history *logs
 		}
 
 		err := w.Write([]string{
-			strconv.Itoa(int(l.Ts.Unix())),
+			strconv.Itoa(int(l.Ts.Time.Unix())),
 			// l.Ts.Format(time.RFC3339),
-			l.Ts.Format("2006-01-02 15:04:05"),
+			l.Ts.Time.Format("2006-01-02 15:04:05"),
 			offset,
 			step,
 			score,
-			fmt.Sprintf("%d", l.MonitorID.Int32),
+			fmt.Sprintf("%d", l.MonitorID.Int64),
 			monName,
 			rtt,
 			leap,
@@ -464,7 +465,7 @@ func setHistoryCacheControl(c echo.Context, history *logscores.LogScoreHistory)
 	if len(history.LogScores) == 0 ||
 		// cache for longer if data hasn't updated for a while; or we didn't
 		// find any.
-		(time.Now().Add(-8 * time.Hour).After(history.LogScores[len(history.LogScores)-1].Ts)) {
+		(time.Now().Add(-8 * time.Hour).After(history.LogScores[len(history.LogScores)-1].Ts.Time)) {
 		hdr.Set("Cache-Control", "s-maxage=260,max-age=360")
 	} else {
 		if len(history.LogScores) == 1 {

server/server.go

@@ -2,17 +2,16 @@ package server
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"golang.org/x/sync/errgroup"
 
+	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/labstack/echo-contrib/echoprometheus"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
@@ -36,7 +35,7 @@ import (
 )
 
 type Server struct {
-	db     *sql.DB
+	db     *pgxpool.Pool
 	ch     *chdb.ClickHouse
 	config *config.Config
 
@@ -55,7 +54,7 @@ func NewServer(ctx context.Context, configFile string) (*Server, error) {
 	}
 	db, err := ntpdb.OpenDB(ctx, configFile)
 	if err != nil {
-		return nil, fmt.Errorf("mysql open: %w", err)
+		return nil, fmt.Errorf("postgres open: %w", err)
 	}
 
 	conf := config.New()
@@ -303,22 +302,9 @@ func healthHandler(srv *Server, log *slog.Logger) func(w http.ResponseWriter, re
 		defer cancel()
 		g, ctx := errgroup.WithContext(ctx)
 
-		stats := srv.db.Stats()
-		if stats.OpenConnections > 3 {
-			log.InfoContext(ctx, "health requests", "url", req.URL.String(), "stats", stats)
-		}
-
-		if resetParam := req.URL.Query().Get("reset"); resetParam != "" {
-			reset, err := strconv.ParseBool(resetParam)
-			log.InfoContext(ctx, "db reset request", "err", err, "reset", reset)
-
-			if err == nil && reset {
-				// this feature was to debug some specific problem
-				log.InfoContext(ctx, "setting idle db conns to zero")
-				srv.db.SetConnMaxLifetime(30 * time.Second)
-				srv.db.SetMaxIdleConns(0)
-				srv.db.SetMaxIdleConns(4)
-			}
+		stats := srv.db.Stat()
+		if stats.TotalConns() > 3 {
+			log.InfoContext(ctx, "health requests", "url", req.URL.String(), "total_conns", stats.TotalConns())
 		}
 
 		g.Go(func() error {
@@ -340,7 +326,7 @@ func healthHandler(srv *Server, log *slog.Logger) func(w http.ResponseWriter, re
 		})
 
 		g.Go(func() error {
-			err := srv.db.PingContext(ctx)
+			err := srv.db.Ping(ctx)
 			if err != nil {
 				log.WarnContext(ctx, "db ping", "err", err)
 				return err

server/zones.go

@@ -1,12 +1,12 @@
 package server
 
 import (
-	"database/sql"
 	"errors"
 	"net/http"
 	"strconv"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/labstack/echo/v4"
 	"go.ntppool.org/common/logger"
 	"go.ntppool.org/common/tracing"
@@ -27,7 +27,7 @@ func (srv *Server) zoneCounts(c echo.Context) error {
 
 	zone, err := q.GetZoneByName(ctx, c.Param("zone_name"))
 	if err != nil || zone.ID == 0 {
-		if errors.Is(err, sql.ErrNoRows) {
+		if errors.Is(err, pgx.ErrNoRows) {
 			return c.String(http.StatusNotFound, "Not found")
 		}
 		log.ErrorContext(ctx, "could not query for zone", "err", err)
@@ -37,7 +37,7 @@ func (srv *Server) zoneCounts(c echo.Context) error {
 
 	counts, err := q.GetZoneCounts(ctx, zone.ID)
 	if err != nil {
-		if !errors.Is(err, sql.ErrNoRows) {
+		if !errors.Is(err, pgx.ErrNoRows) {
 			log.ErrorContext(ctx, "get counts", "err", err)
 			span.RecordError(err)
 			return c.String(http.StatusInternalServerError, "internal error")
@@ -71,7 +71,7 @@ func (srv *Server) zoneCounts(c echo.Context) error {
 		count := 0
 		dates := map[int64]bool{}
 		for _, c := range counts {
-			ep := c.Date.Unix()
+			ep := c.Date.Time.Unix()
 			if _, ok := dates[ep]; !ok {
 				count++
 				dates[ep] = true
@@ -99,13 +99,13 @@ func (srv *Server) zoneCounts(c echo.Context) error {
 	lastSkip := int64(0)
 	skipThreshold := 0.5
 	for _, c := range counts {
-		cDate := c.Date.Unix()
+		cDate := c.Date.Time.Unix()
 		if (toSkip <= skipThreshold && cDate != lastSkip) ||
 			lastDate == cDate ||
 			mostRecentDate == cDate {
-			// log.Info("adding date", "date", c.Date.Format(time.DateOnly))
+			// log.Info("adding date", "date", c.Date.Time.Format(time.DateOnly))
 			rv.History = append(rv.History, historyEntry{
-				D:  c.Date.Format(time.DateOnly),
+				D:  c.Date.Time.Format(time.DateOnly),
 				Ts: int(cDate),
 				Ac: int(c.CountActive),
 				Rc: int(c.CountRegistered),