scores: clickhouse support

chdb/chdnsanswers.go

@@ -36,7 +36,7 @@ func (d *ClickHouse) ServerAnswerCounts(ctx context.Context, serverIP string, da
 	ctx, span := tracing.Tracer().Start(ctx, "ServerAnswerCounts")
 	defer span.End()
 
-	conn := d.conn
+	conn := d.Logs
 
 	log := logger.Setup().With("server", serverIP)
 
@@ -100,7 +100,7 @@ func (d *ClickHouse) AnswerTotals(ctx context.Context, qtype string, days int) (
 	defer span.End()
 
 	// queries by UserCC / Qtype for the ServerIP
-	rows, err := d.conn.Query(clickhouse.Context(ctx,
+	rows, err := d.Logs.Query(clickhouse.Context(ctx,
 		clickhouse.WithSpan(span.SpanContext()),
 	), `
 	select UserCC,Qtype,sum(queries) as queries

chdb/db.go

@@ -2,35 +2,82 @@ package chdb
 
 import (
 	"context"
+	"os"
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
-	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"gopkg.in/yaml.v3"
 
 	"go.ntppool.org/common/logger"
 	"go.ntppool.org/common/version"
 )
 
+type Config struct {
+	ClickHouse struct {
+		Scores DBConfig `yaml:"scores"`
+		Logs   DBConfig `yaml:"logs"`
+	} `yaml:"clickhouse"`
+}
+
+type DBConfig struct {
+	Host     string
+	Database string
+}
+
 type ClickHouse struct {
-	conn clickhouse.Conn
+	Logs   clickhouse.Conn
+	Scores clickhouse.Conn
 }
 
 func New(ctx context.Context, dbConfigPath string) (*ClickHouse, error) {
-	conn, err := setupClickhouse(ctx)
+	ch, err := setupClickhouse(ctx, dbConfigPath)
 	if err != nil {
 		return nil, err
 	}
-	return &ClickHouse{conn: conn}, nil
+	return ch, nil
 }
 
-func setupClickhouse(ctx context.Context) (driver.Conn, error) {
+func setupClickhouse(ctx context.Context, configFile string) (*ClickHouse, error) {
 
+	log := logger.FromContext(ctx)
+
+	log.InfoContext(ctx, "opening config", "file", configFile)
+
+	dbFile, err := os.Open(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	dec := yaml.NewDecoder(dbFile)
+
+	cfg := Config{}
+
+	err = dec.Decode(&cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := &ClickHouse{}
+
+	ch.Logs, err = open(ctx, cfg.ClickHouse.Logs)
+	if err != nil {
+		return nil, err
+	}
+	ch.Scores, err = open(ctx, cfg.ClickHouse.Scores)
+	if err != nil {
+		return nil, err
+	}
+
+	return ch, nil
+}
+
+func open(ctx context.Context, cfg DBConfig) (clickhouse.Conn, error) {
 	log := logger.Setup()
 
 	conn, err := clickhouse.Open(&clickhouse.Options{
-		Addr: []string{"10.43.207.123:9000"},
+		Addr: []string{cfg.Host + ":9000"},
 		Auth: clickhouse.Auth{
-			Database: "geodns3",
+			Database: cfg.Database,
 			Username: "default",
 			Password: "",
 		},

chdb/geoqueries.go

@@ -36,7 +36,7 @@ func (d *ClickHouse) UserCountryData(ctx context.Context) (*UserCountry, error)
 	ctx, span := tracing.Tracer().Start(ctx, "UserCountryData")
 	defer span.End()
 
-	rows, err := d.conn.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())),
+	rows, err := d.Logs.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())),
 		"select max(dt) as d,UserCC,Qtype,sum(queries) as queries from by_usercc_1d where dt > now() - INTERVAL 4 DAY group by rollup(Qtype,UserCC) order by UserCC,Qtype;")
 	if err != nil {
 		log.ErrorContext(ctx, "query error", "err", err)

chdb/logscores.go

@@ -0,0 +1,86 @@
+package chdb
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"go.ntppool.org/common/logger"
+	"go.ntppool.org/common/tracing"
+	"go.ntppool.org/data-api/ntpdb"
+)
+
+func (d *ClickHouse) Logscores(ctx context.Context, serverID, monitorID int, since time.Time, limit int) ([]ntpdb.LogScore, error) {
+	log := logger.Setup()
+	ctx, span := tracing.Tracer().Start(ctx, "CH Logscores")
+	defer span.End()
+
+	if since.IsZero() {
+		since = time.Now().Add(4 * -24 * time.Hour)
+	}
+
+	args := []interface{}{serverID, since, limit}
+	query := `select id,monitor_id,server_id,ts,
+                toFloat64(score),toFloat64(step),offset,
+                rtt,leap,warning,error
+              from log_scores
+              where
+                server_id = ?
+                and ts > ?
+              order by ts desc
+              limit ?;`
+
+	if monitorID > 0 {
+		query = `select id,monitor_id,server_id,ts,
+                toFloat64(score),toFloat64(step),offset,
+                rtt,leap,warning,error
+              from log_scores
+              where
+                server_id = ?
+                and monitor_id = ?
+                and ts > ?
+              order by ts desc
+              limit ?;`
+		args = []interface{}{serverID, monitorID, since, limit}
+	}
+
+	rows, err := d.Scores.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())),
+		query, args...,
+	)
+	if err != nil {
+		log.ErrorContext(ctx, "query error", "err", err)
+		return nil, fmt.Errorf("database error")
+	}
+
+	rv := []ntpdb.LogScore{}
+
+	for rows.Next() {
+
+		row := ntpdb.LogScore{}
+
+		if err := rows.Scan(
+			&row.ID,
+			&row.MonitorID,
+			&row.ServerID,
+			&row.Ts,
+			&row.Score,
+			&row.Step,
+			&row.Offset,
+			&row.Rtt,
+			&row.Attributes.Leap,
+			&row.Attributes.Warning,
+			&row.Attributes.Error,
+		); err != nil {
+			log.Error("could not parse row", "err", err)
+			continue
+		}
+
+		rv = append(rv, row)
+
+	}
+
+	// log.InfoContext(ctx, "returning data", "rv", rv)
+
+	return rv, nil
+}

go.mod

@@ -9,28 +9,31 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/labstack/echo/v4 v4.11.4
-	github.com/samber/slog-echo v1.11.0
+	github.com/samber/slog-echo v1.12.1
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
-	go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511
-	go.ntppool.org/common v0.2.6
-	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
-	go.opentelemetry.io/otel v1.21.0
-	go.opentelemetry.io/otel/trace v1.21.0
+	go.ntppool.org/api v0.1.8
+	go.ntppool.org/common v0.2.7
+	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
+	go.opentelemetry.io/otel v1.22.0
+	go.opentelemetry.io/otel/trace v1.22.0
 	golang.org/x/sync v0.6.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/ClickHouse/ch-go v0.61.1 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/abh/certman v0.4.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -42,6 +45,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/labstack/echo-contrib v0.15.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -62,20 +66,20 @@ require (
 	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
+	go.opentelemetry.io/otel/metric v1.22.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.22.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
 	google.golang.org/grpc v1.60.1 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
 )

go.sum

@@ -9,6 +9,10 @@ github.com/ClickHouse/clickhouse-go/v2 v2.17.0 h1:xvsVYxOWb2obaIwL9NJZSZ3T/umJSh
 github.com/ClickHouse/clickhouse-go/v2 v2.17.0/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ=
 github.com/ClickHouse/clickhouse-go/v2 v2.17.1 h1:ZCmAYWpu75IyEi7+Yrs/uaAjiCGY5wfW5kXo64exkX4=
 github.com/ClickHouse/clickhouse-go/v2 v2.17.1/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ=
+github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
+github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
+github.com/abh/certman v0.4.0 h1:XHoDtb0YyRQPclaHMrBDlKTVZpNjTK6vhB0S3Bd/Sbs=
+github.com/abh/certman v0.4.0/go.mod h1:x8QhpKVZifmV1Hdiwdg9gLo2GMPAxezz1s3zrVnPs+I=
 github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
@@ -25,6 +29,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
 github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
 github.com/go-faster/errors v0.7.0 h1:UnD/xusnfUgtEYkgRZohqL2AfmPTwv13NAJwwFFaNYc=
@@ -80,6 +86,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/labstack/echo-contrib v0.15.0 h1:9K+oRU265y4Mu9zpRDv3X+DGTqUALY6oRHCSZZKCRVU=
+github.com/labstack/echo-contrib v0.15.0/go.mod h1:lei+qt5CLB4oa7VHTE0yEfQSEB9XTJI1LUqko9UWvo4=
 github.com/labstack/echo/v4 v4.11.3 h1:Upyu3olaqSHkCjs1EJJwQ3WId8b8b1hxbogyommKktM=
 github.com/labstack/echo/v4 v4.11.3/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws=
 github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8=
@@ -137,6 +145,8 @@ github.com/samber/slog-echo v1.9.1 h1:QEzOuZtZiXe0/60bmfcVZdwYzq1T6SCBC4RiMfg9Ri
 github.com/samber/slog-echo v1.9.1/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ=
 github.com/samber/slog-echo v1.11.0 h1:qxj2KBeGfD4xW1UXPVElV4QROn+ZxNqM95U3bwq7dC0=
 github.com/samber/slog-echo v1.11.0/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ=
+github.com/samber/slog-echo v1.12.1 h1:TgpA4luiqe2sa6SDsraM1hUcLAXF8ZofjXM0SzIJkJc=
+github.com/samber/slog-echo v1.12.1/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
@@ -174,6 +184,8 @@ go.ntppool.org/api v0.1.8-0.20231209235224-e3d38c8b1c15 h1:xzvsIUwRajlVQsRRrXjkl
 go.ntppool.org/api v0.1.8-0.20231209235224-e3d38c8b1c15/go.mod h1:9FKbwWfF7eRU7GZVEI3wUYv71ZKt16cLCGfxLayzS2Q=
 go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511 h1:ilTOX5NQcdDiNohhDexUevUrhDFDOrlB7uCM0ilQsLs=
 go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511/go.mod h1:9FKbwWfF7eRU7GZVEI3wUYv71ZKt16cLCGfxLayzS2Q=
+go.ntppool.org/api v0.1.8 h1:s6sOgpPsJAsykp6m5tQaDT4ovD5YMLOl2ywzTO675RM=
+go.ntppool.org/api v0.1.8/go.mod h1:OPogxxgiLnOP0yKnLqTqv0dByPengrRTdlF5HitjtpM=
 go.ntppool.org/common v0.2.5 h1:fvuFrCCbmaRzZOSwv71+yhqVLOTDB/fD7YlscdGa6qs=
 go.ntppool.org/common v0.2.5/go.mod h1:Cw8mq8jd2sLCxbTNzYXKXn3qKo2ZLERZ6V/eLcSgDHw=
 go.ntppool.org/common v0.2.6-0.20231211031613-608f05d39551 h1:kJdF3U4KBuJJtbF04d5OA/QttxDY/fOgHe1oUDEgLGA=
@@ -182,26 +194,46 @@ go.ntppool.org/common v0.2.6-0.20231211044338-5c7ae6ab8ac9 h1:5NHaULU9+qu6hA/teq
 go.ntppool.org/common v0.2.6-0.20231211044338-5c7ae6ab8ac9/go.mod h1:Cw8mq8jd2sLCxbTNzYXKXn3qKo2ZLERZ6V/eLcSgDHw=
 go.ntppool.org/common v0.2.6 h1:lOBkTTJYKPGPHwsmqcGOrH4PTIBMo+7mgb9fGsWgdjg=
 go.ntppool.org/common v0.2.6/go.mod h1:pqJD5pKizHE/a4BmvVdg4ne6LJAiwluaF19gzWcwp64=
+go.ntppool.org/common v0.2.7 h1:TaqyOgMTXrLm3b2srB6aMBBbIM2Zt7+jB6uEUV6fOfE=
+go.ntppool.org/common v0.2.7/go.mod h1:7gNYzGKJyVDo3g4NDH96ENTu3LVFJXyumNGAgmQkkrY=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1 h1:yJWyqeE+8jdOJpt+ZFn7sX05EJAK/9C4jjNZyb61xZg=
 go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1/go.mod h1:tlgpIvi6LCv4QIZQyBc8Gkr6HDxbJLTh9eQPNZAaljE=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0 h1:LxU1CtJeUgR3sSIoEqTWuJ1VFAgybxpqKZjeTAFvDfo=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0/go.mod h1:kNOJ6ovdGbJ/L8Oq4+5yftrkp78Z8V4M8H9aJcMe46w=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0 h1:rw+yB4sMhufNzbVHGG9SDMSrw1CKSnRqfjJnMpAH4dE=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0/go.mod h1:2NonlJyJNVbDK/hCwiLsu5gsD2bVtmIzQ/tGzWq58us=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
 go.opentelemetry.io/contrib/propagators/b3 v1.21.1 h1:WPYiUgmw3+b7b3sQ1bFBFAf0q+Di9dvNc3AtYfnT4RQ=
 go.opentelemetry.io/contrib/propagators/b3 v1.21.1/go.mod h1:EmzokPoSqsYMBVK4nRnhsfm5mbn8J1eDuz/U1UaQaWg=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
+go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
+go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY=
 go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
+go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
+go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
+go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
 go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
+go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
+go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI=
@@ -226,6 +258,8 @@ golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBi
 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
+golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
@@ -286,6 +320,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 h1:
 google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg=
 google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac h1:OZkkudMUu9LVQMCoRUbI/1p5VCo9BOrlvkqMvWtqa6s=
 google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg=
+google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU=
+google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU=
@@ -294,6 +330,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s=
 google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k=

logscores/history.go

@@ -7,6 +7,7 @@ import (
 
 	"go.ntppool.org/common/logger"
 	"go.ntppool.org/common/tracing"
+	"go.ntppool.org/data-api/chdb"
 	"go.ntppool.org/data-api/ntpdb"
 	"go.opentelemetry.io/otel/attribute"
 )
@@ -14,24 +15,53 @@ import (
 type LogScoreHistory struct {
 	LogScores []ntpdb.LogScore
 	Monitors  map[int]string
-	MonitorIDs []uint32
+	// MonitorIDs []uint32
 }
 
-func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) {
-	log := logger.Setup()
-	ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistory")
+func GetHistoryClickHouse(ctx context.Context, ch *chdb.ClickHouse, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) {
+	log := logger.FromContext(ctx)
+	ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistoryClickHouse")
 	defer span.End()
 
-	if count == 0 {
-		count = 200
-	}
-
 	span.SetAttributes(
 		attribute.Int("server", int(serverID)),
 		attribute.Int("monitor", int(monitorID)),
 	)
 
-	log.Debug("GetHistory", "server", serverID, "monitor", monitorID, "since", since, "count", count)
+	log.Debug("GetHistoryCH", "server", serverID, "monitor", monitorID, "since", since, "count", count)
+
+	ls, err := ch.Logscores(ctx, int(serverID), int(monitorID), since, count)
+
+	if err != nil {
+		log.ErrorContext(ctx, "clickhouse logscores", "err", err)
+		return nil, err
+	}
+
+	q := ntpdb.NewWrappedQuerier(ntpdb.New(db))
+
+	monitors, err := getMonitorNames(ctx, ls, q)
+	if err != nil {
+		return nil, err
+	}
+
+	return &LogScoreHistory{
+		LogScores: ls,
+		Monitors:  monitors,
+		// MonitorIDs: monitorIDs,
+	}, nil
+}
+
+func GetHistoryMySQL(ctx context.Context, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) {
+	log := logger.FromContext(ctx)
+	ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistoryMySQL")
+	defer span.End()
+
+	span.SetAttributes(
+		attribute.Int("server", int(serverID)),
+		attribute.Int("monitor", int(monitorID)),
+	)
+
+	log.Debug("GetHistoryMySQL", "server", serverID, "monitor", monitorID, "since", since, "count", count)
 
 	q := ntpdb.NewWrappedQuerier(ntpdb.New(db))
 
@@ -53,6 +83,19 @@ func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, sin
 		return nil, err
 	}
 
+	monitors, err := getMonitorNames(ctx, ls, q)
+	if err != nil {
+		return nil, err
+	}
+
+	return &LogScoreHistory{
+		LogScores: ls,
+		Monitors:  monitors,
+		// MonitorIDs: monitorIDs,
+	}, nil
+}
+
+func getMonitorNames(ctx context.Context, ls []ntpdb.LogScore, q ntpdb.QuerierTx) (map[int]string, error) {
 	monitors := map[int]string{}
 	monitorIDs := []uint32{}
 	for _, l := range ls {
@@ -73,10 +116,5 @@ func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, sin
 	for _, m := range dbmons {
 		monitors[int(m.ID)] = m.DisplayName()
 	}
-
-	return &LogScoreHistory{
-		LogScores:  ls,
-		Monitors:   monitors,
-		MonitorIDs: monitorIDs,
-	}, nil
+	return monitors, nil
 }

server/history.go

@@ -42,8 +42,17 @@ func paramHistoryMode(s string) historyMode {
 	}
 }
 
-func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb.Server) (*logscores.LogScoreHistory, error) {
-	log := logger.Setup()
+type historyParameters struct {
+	limit     int
+	monitorID int
+	server    ntpdb.Server
+	since     time.Time
+}
+
+func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context) (historyParameters, error) {
+	log := logger.FromContext(ctx)
+
+	p := historyParameters{}
 
 	limit := 0
 	if limitParam, err := strconv.Atoi(c.QueryParam("limit")); err == nil {
@@ -51,20 +60,17 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb.
 	} else {
 		limit = 100
 	}
+
 	if limit > 10000 {
 		limit = 10000
 	}
 
-	since, _ := strconv.ParseInt(c.QueryParam("since"), 10, 64) // defaults to 0 so don't care if it parses
-
-	monitorParam := c.QueryParam("monitor")
-
-	if since > 0 {
-		c.Request().Header.Set("Cache-Control", "s-maxage=300")
-	}
+	p.limit = limit
 
 	q := ntpdb.NewWrappedQuerier(ntpdb.New(srv.db))
 
+	monitorParam := c.QueryParam("monitor")
+
 	var monitorID uint32 = 0
 	switch monitorParam {
 	case "":
@@ -84,7 +90,7 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb.
 			// only accept the name prefix; no wildcards; trust the database
 			// to filter out any other crazy
 			if strings.ContainsAny(monitorParam, "_%. \t\n") {
-				return nil, echo.NewHTTPError(http.StatusNotFound, "monitor not found")
+				return p, echo.NewHTTPError(http.StatusNotFound, "monitor not found")
 			}
 
 			if err != nil {
@@ -92,22 +98,33 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb.
 				monitor, err := q.GetMonitorByName(ctx, sql.NullString{Valid: true, String: monitorParam})
 				if err != nil {
 					log.Warn("could not find monitor", "name", monitorParam, "err", err)
-					return nil, echo.NewHTTPError(http.StatusNotFound, "monitor not found")
+					return p, echo.NewHTTPError(http.StatusNotFound, "monitor not found")
 				}
 				monitorID = monitor.ID
 			}
 		}
 	}
 
+	p.monitorID = int(monitorID)
 	log.DebugContext(ctx, "monitor param", "monitor", monitorID)
 
-	sinceTime := time.Unix(since, 0)
+	since, _ := strconv.ParseInt(c.QueryParam("since"), 10, 64) // defaults to 0 so don't care if it parses
 	if since > 0 {
-		log.Warn("monitor data requested with since parameter, not supported", "since", sinceTime)
+		p.since = time.Unix(since, 0)
+	}
+	if !p.since.IsZero() {
+		log.Warn("monitor data requested with since parameter", "since", p.since)
 	}
 
-	ls, err := logscores.GetHistory(ctx, srv.db, server.ID, monitorID, sinceTime, limit)
+	return p, nil
+}
 
+func (srv *Server) getHistoryCH(ctx context.Context, c echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) {
+	return logscores.GetHistoryClickHouse(ctx, srv.ch, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit)
+}
+
+func (srv *Server) getHistoryMySQL(ctx context.Context, c echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) {
+	ls, err := logscores.GetHistoryMySQL(ctx, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit)
 	return ls, err
 }
 
@@ -124,6 +141,13 @@ func (srv *Server) history(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusNotFound, "invalid mode")
 	}
 
+	p, err := srv.getHistoryParameters(ctx, c)
+	if err != nil {
+		log.Error("get history parameters", "err", err)
+		span.RecordError(err)
+		return echo.NewHTTPError(http.StatusInternalServerError, "internal error")
+	}
+
 	server, err := srv.FindServer(ctx, c.Param("server"))
 	if err != nil {
 		log.Error("find server", "err", err)
@@ -139,7 +163,15 @@ func (srv *Server) history(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusNotFound, "server not found")
 	}
 
-	history, err := srv.getHistory(ctx, c, server)
+	p.server = server
+
+	var history *logscores.LogScoreHistory
+
+	if c.QueryParam("source") == "c" {
+		history, err = srv.getHistoryCH(ctx, c, p)
+	} else {
+		history, err = srv.getHistoryMySQL(ctx, c, p)
+	}
 	if err != nil {
 		var httpError *echo.HTTPError
 		if errors.As(err, &httpError) {
@@ -202,10 +234,15 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb
 
 	// log.InfoContext(ctx, "monitor id list", "ids", history.MonitorIDs)
 
+	monitorIDs := []uint32{}
+	for k := range history.Monitors {
+		monitorIDs = append(monitorIDs, uint32(k))
+	}
+
 	q := ntpdb.NewWrappedQuerier(ntpdb.New(srv.db))
 	logScoreMonitors, err := q.GetServerScores(ctx,
 		ntpdb.GetServerScoresParams{
-			MonitorIDs: history.MonitorIDs,
+			MonitorIDs: monitorIDs,
 			ServerID:   server.ID,
 		},
 	)

server/server.go

@@ -174,6 +174,7 @@ func (srv *Server) Run() error {
 	e.Use(middleware.RecoverWithConfig(middleware.RecoverConfig{
 		LogErrorFunc: func(c echo.Context, err error, stack []byte) error {
 			log.ErrorContext(c.Request().Context(), err.Error(), "stack", string(stack))
+			fmt.Println(string(stack))
 			return err
 		},
 	}))