diff --git a/chdb/chdnsanswers.go b/chdb/chdnsanswers.go index 3ed6c69..4c18e1a 100644 --- a/chdb/chdnsanswers.go +++ b/chdb/chdnsanswers.go @@ -36,7 +36,7 @@ func (d *ClickHouse) ServerAnswerCounts(ctx context.Context, serverIP string, da ctx, span := tracing.Tracer().Start(ctx, "ServerAnswerCounts") defer span.End() - conn := d.conn + conn := d.Logs log := logger.Setup().With("server", serverIP) @@ -100,7 +100,7 @@ func (d *ClickHouse) AnswerTotals(ctx context.Context, qtype string, days int) ( defer span.End() // queries by UserCC / Qtype for the ServerIP - rows, err := d.conn.Query(clickhouse.Context(ctx, + rows, err := d.Logs.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext()), ), ` select UserCC,Qtype,sum(queries) as queries diff --git a/chdb/db.go b/chdb/db.go index fff08ac..343ef1a 100644 --- a/chdb/db.go +++ b/chdb/db.go @@ -2,35 +2,82 @@ package chdb import ( "context" + "os" "time" "github.com/ClickHouse/clickhouse-go/v2" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "gopkg.in/yaml.v3" "go.ntppool.org/common/logger" "go.ntppool.org/common/version" ) +type Config struct { + ClickHouse struct { + Scores DBConfig `yaml:"scores"` + Logs DBConfig `yaml:"logs"` + } `yaml:"clickhouse"` +} + +type DBConfig struct { + Host string + Database string +} + type ClickHouse struct { - conn clickhouse.Conn + Logs clickhouse.Conn + Scores clickhouse.Conn } func New(ctx context.Context, dbConfigPath string) (*ClickHouse, error) { - conn, err := setupClickhouse(ctx) + ch, err := setupClickhouse(ctx, dbConfigPath) if err != nil { return nil, err } - return &ClickHouse{conn: conn}, nil + return ch, nil } -func setupClickhouse(ctx context.Context) (driver.Conn, error) { +func setupClickhouse(ctx context.Context, configFile string) (*ClickHouse, error) { + log := logger.FromContext(ctx) + + log.InfoContext(ctx, "opening config", "file", configFile) + + dbFile, err := os.Open(configFile) + if err != nil { + return nil, err + } + + dec := yaml.NewDecoder(dbFile) + + cfg := Config{} + + err = dec.Decode(&cfg) + if err != nil { + return nil, err + } + + ch := &ClickHouse{} + + ch.Logs, err = open(ctx, cfg.ClickHouse.Logs) + if err != nil { + return nil, err + } + ch.Scores, err = open(ctx, cfg.ClickHouse.Scores) + if err != nil { + return nil, err + } + + return ch, nil +} + +func open(ctx context.Context, cfg DBConfig) (clickhouse.Conn, error) { log := logger.Setup() conn, err := clickhouse.Open(&clickhouse.Options{ - Addr: []string{"10.43.207.123:9000"}, + Addr: []string{cfg.Host + ":9000"}, Auth: clickhouse.Auth{ - Database: "geodns3", + Database: cfg.Database, Username: "default", Password: "", }, diff --git a/chdb/geoqueries.go b/chdb/geoqueries.go index 73f5239..e324e6a 100644 --- a/chdb/geoqueries.go +++ b/chdb/geoqueries.go @@ -36,7 +36,7 @@ func (d *ClickHouse) UserCountryData(ctx context.Context) (*UserCountry, error) ctx, span := tracing.Tracer().Start(ctx, "UserCountryData") defer span.End() - rows, err := d.conn.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())), + rows, err := d.Logs.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())), "select max(dt) as d,UserCC,Qtype,sum(queries) as queries from by_usercc_1d where dt > now() - INTERVAL 4 DAY group by rollup(Qtype,UserCC) order by UserCC,Qtype;") if err != nil { log.ErrorContext(ctx, "query error", "err", err) diff --git a/chdb/logscores.go b/chdb/logscores.go new file mode 100644 index 0000000..73fbc87 --- /dev/null +++ b/chdb/logscores.go @@ -0,0 +1,86 @@ +package chdb + +import ( + "context" + "fmt" + "time" + + "github.com/ClickHouse/clickhouse-go/v2" + "go.ntppool.org/common/logger" + "go.ntppool.org/common/tracing" + "go.ntppool.org/data-api/ntpdb" +) + +func (d *ClickHouse) Logscores(ctx context.Context, serverID, monitorID int, since time.Time, limit int) ([]ntpdb.LogScore, error) { + log := logger.Setup() + ctx, span := tracing.Tracer().Start(ctx, "CH Logscores") + defer span.End() + + if since.IsZero() { + since = time.Now().Add(4 * -24 * time.Hour) + } + + args := []interface{}{serverID, since, limit} + query := `select id,monitor_id,server_id,ts, + toFloat64(score),toFloat64(step),offset, + rtt,leap,warning,error + from log_scores + where + server_id = ? + and ts > ? + order by ts desc + limit ?;` + + if monitorID > 0 { + query = `select id,monitor_id,server_id,ts, + toFloat64(score),toFloat64(step),offset, + rtt,leap,warning,error + from log_scores + where + server_id = ? + and monitor_id = ? + and ts > ? + order by ts desc + limit ?;` + args = []interface{}{serverID, monitorID, since, limit} + } + + rows, err := d.Scores.Query(clickhouse.Context(ctx, clickhouse.WithSpan(span.SpanContext())), + query, args..., + ) + if err != nil { + log.ErrorContext(ctx, "query error", "err", err) + return nil, fmt.Errorf("database error") + } + + rv := []ntpdb.LogScore{} + + for rows.Next() { + + row := ntpdb.LogScore{} + + if err := rows.Scan( + &row.ID, + &row.MonitorID, + &row.ServerID, + &row.Ts, + &row.Score, + &row.Step, + &row.Offset, + &row.Rtt, + &row.Attributes.Leap, + &row.Attributes.Warning, + &row.Attributes.Error, + ); err != nil { + log.Error("could not parse row", "err", err) + continue + } + + rv = append(rv, row) + + } + + // log.InfoContext(ctx, "returning data", "rv", rv) + + return rv, nil +} diff --git a/go.mod b/go.mod index 5cfc9f8..3881e25 100644 --- a/go.mod +++ b/go.mod @@ -9,28 +9,31 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/labstack/echo/v4 v4.11.4 - github.com/samber/slog-echo v1.11.0 + github.com/samber/slog-echo v1.12.1 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 - go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511 - go.ntppool.org/common v0.2.6 - go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1 - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 - go.opentelemetry.io/otel v1.21.0 - go.opentelemetry.io/otel/trace v1.21.0 + go.ntppool.org/api v0.1.8 + go.ntppool.org/common v0.2.7 + go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 + go.opentelemetry.io/otel v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 golang.org/x/sync v0.6.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/ClickHouse/ch-go v0.61.1 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/abh/certman v0.4.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -42,6 +45,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.4 // indirect + github.com/labstack/echo-contrib v0.15.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -62,20 +66,20 @@ require ( github.com/stretchr/objx v0.5.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect golang.org/x/crypto v0.18.0 // indirect - golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect + golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.32.0 // indirect ) diff --git a/go.sum b/go.sum index 321a805..6bcf900 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,10 @@ github.com/ClickHouse/clickhouse-go/v2 v2.17.0 h1:xvsVYxOWb2obaIwL9NJZSZ3T/umJSh github.com/ClickHouse/clickhouse-go/v2 v2.17.0/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ= github.com/ClickHouse/clickhouse-go/v2 v2.17.1 h1:ZCmAYWpu75IyEi7+Yrs/uaAjiCGY5wfW5kXo64exkX4= github.com/ClickHouse/clickhouse-go/v2 v2.17.1/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/abh/certman v0.4.0 h1:XHoDtb0YyRQPclaHMrBDlKTVZpNjTK6vhB0S3Bd/Sbs= +github.com/abh/certman v0.4.0/go.mod h1:x8QhpKVZifmV1Hdiwdg9gLo2GMPAxezz1s3zrVnPs+I= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= @@ -25,6 +29,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.0 h1:UnD/xusnfUgtEYkgRZohqL2AfmPTwv13NAJwwFFaNYc= @@ -80,6 +86,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo-contrib v0.15.0 h1:9K+oRU265y4Mu9zpRDv3X+DGTqUALY6oRHCSZZKCRVU= +github.com/labstack/echo-contrib v0.15.0/go.mod h1:lei+qt5CLB4oa7VHTE0yEfQSEB9XTJI1LUqko9UWvo4= github.com/labstack/echo/v4 v4.11.3 h1:Upyu3olaqSHkCjs1EJJwQ3WId8b8b1hxbogyommKktM= github.com/labstack/echo/v4 v4.11.3/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws= github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= @@ -137,6 +145,8 @@ github.com/samber/slog-echo v1.9.1 h1:QEzOuZtZiXe0/60bmfcVZdwYzq1T6SCBC4RiMfg9Ri github.com/samber/slog-echo v1.9.1/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ= github.com/samber/slog-echo v1.11.0 h1:qxj2KBeGfD4xW1UXPVElV4QROn+ZxNqM95U3bwq7dC0= github.com/samber/slog-echo v1.11.0/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ= +github.com/samber/slog-echo v1.12.1 h1:TgpA4luiqe2sa6SDsraM1hUcLAXF8ZofjXM0SzIJkJc= +github.com/samber/slog-echo v1.12.1/go.mod h1:/f78pHjVxGrIlHlS5fzWiW+BxkWltQ+SWKk8LKMjAMQ= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -174,6 +184,8 @@ go.ntppool.org/api v0.1.8-0.20231209235224-e3d38c8b1c15 h1:xzvsIUwRajlVQsRRrXjkl go.ntppool.org/api v0.1.8-0.20231209235224-e3d38c8b1c15/go.mod h1:9FKbwWfF7eRU7GZVEI3wUYv71ZKt16cLCGfxLayzS2Q= go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511 h1:ilTOX5NQcdDiNohhDexUevUrhDFDOrlB7uCM0ilQsLs= go.ntppool.org/api v0.1.8-0.20231210025001-f2c143296511/go.mod h1:9FKbwWfF7eRU7GZVEI3wUYv71ZKt16cLCGfxLayzS2Q= +go.ntppool.org/api v0.1.8 h1:s6sOgpPsJAsykp6m5tQaDT4ovD5YMLOl2ywzTO675RM= +go.ntppool.org/api v0.1.8/go.mod h1:OPogxxgiLnOP0yKnLqTqv0dByPengrRTdlF5HitjtpM= go.ntppool.org/common v0.2.5 h1:fvuFrCCbmaRzZOSwv71+yhqVLOTDB/fD7YlscdGa6qs= go.ntppool.org/common v0.2.5/go.mod h1:Cw8mq8jd2sLCxbTNzYXKXn3qKo2ZLERZ6V/eLcSgDHw= go.ntppool.org/common v0.2.6-0.20231211031613-608f05d39551 h1:kJdF3U4KBuJJtbF04d5OA/QttxDY/fOgHe1oUDEgLGA= @@ -182,26 +194,46 @@ go.ntppool.org/common v0.2.6-0.20231211044338-5c7ae6ab8ac9 h1:5NHaULU9+qu6hA/teq go.ntppool.org/common v0.2.6-0.20231211044338-5c7ae6ab8ac9/go.mod h1:Cw8mq8jd2sLCxbTNzYXKXn3qKo2ZLERZ6V/eLcSgDHw= go.ntppool.org/common v0.2.6 h1:lOBkTTJYKPGPHwsmqcGOrH4PTIBMo+7mgb9fGsWgdjg= go.ntppool.org/common v0.2.6/go.mod h1:pqJD5pKizHE/a4BmvVdg4ne6LJAiwluaF19gzWcwp64= +go.ntppool.org/common v0.2.7 h1:TaqyOgMTXrLm3b2srB6aMBBbIM2Zt7+jB6uEUV6fOfE= +go.ntppool.org/common v0.2.7/go.mod h1:7gNYzGKJyVDo3g4NDH96ENTu3LVFJXyumNGAgmQkkrY= go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1 h1:yJWyqeE+8jdOJpt+ZFn7sX05EJAK/9C4jjNZyb61xZg= go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.46.1/go.mod h1:tlgpIvi6LCv4QIZQyBc8Gkr6HDxbJLTh9eQPNZAaljE= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0 h1:LxU1CtJeUgR3sSIoEqTWuJ1VFAgybxpqKZjeTAFvDfo= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.47.0/go.mod h1:kNOJ6ovdGbJ/L8Oq4+5yftrkp78Z8V4M8H9aJcMe46w= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0 h1:rw+yB4sMhufNzbVHGG9SDMSrw1CKSnRqfjJnMpAH4dE= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.47.0/go.mod h1:2NonlJyJNVbDK/hCwiLsu5gsD2bVtmIzQ/tGzWq58us= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/contrib/propagators/b3 v1.21.1 h1:WPYiUgmw3+b7b3sQ1bFBFAf0q+Di9dvNc3AtYfnT4RQ= go.opentelemetry.io/contrib/propagators/b3 v1.21.1/go.mod h1:EmzokPoSqsYMBVK4nRnhsfm5mbn8J1eDuz/U1UaQaWg= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= @@ -226,6 +258,8 @@ golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBi golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= @@ -286,6 +320,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg= google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac h1:OZkkudMUu9LVQMCoRUbI/1p5VCo9BOrlvkqMvWtqa6s= google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= @@ -294,6 +330,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= diff --git a/logscores/history.go b/logscores/history.go index b34662b..428bb70 100644 --- a/logscores/history.go +++ b/logscores/history.go @@ -7,31 +7,61 @@ import ( "go.ntppool.org/common/logger" "go.ntppool.org/common/tracing" + "go.ntppool.org/data-api/chdb" "go.ntppool.org/data-api/ntpdb" "go.opentelemetry.io/otel/attribute" ) type LogScoreHistory struct { - LogScores []ntpdb.LogScore - Monitors map[int]string - MonitorIDs []uint32 + LogScores []ntpdb.LogScore + Monitors map[int]string + // MonitorIDs []uint32 } -func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) { - log := logger.Setup() - ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistory") +func GetHistoryClickHouse(ctx context.Context, ch *chdb.ClickHouse, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) { + log := logger.FromContext(ctx) + ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistoryClickHouse") defer span.End() - if count == 0 { - count = 200 - } - span.SetAttributes( attribute.Int("server", int(serverID)), attribute.Int("monitor", int(monitorID)), ) - log.Debug("GetHistory", "server", serverID, "monitor", monitorID, "since", since, "count", count) + log.Debug("GetHistoryCH", "server", serverID, "monitor", monitorID, "since", since, "count", count) + + ls, err := ch.Logscores(ctx, int(serverID), int(monitorID), since, count) + + if err != nil { + log.ErrorContext(ctx, "clickhouse logscores", "err", err) + return nil, err + } + + q := ntpdb.NewWrappedQuerier(ntpdb.New(db)) + + monitors, err := getMonitorNames(ctx, ls, q) + if err != nil { + return nil, err + } + + return &LogScoreHistory{ + LogScores: ls, + Monitors: monitors, + // MonitorIDs: monitorIDs, + }, nil +} + +func GetHistoryMySQL(ctx context.Context, db *sql.DB, serverID, monitorID uint32, since time.Time, count int) (*LogScoreHistory, error) { + log := logger.FromContext(ctx) + ctx, span := tracing.Tracer().Start(ctx, "logscores.GetHistoryMySQL") + defer span.End() + + span.SetAttributes( + attribute.Int("server", int(serverID)), + attribute.Int("monitor", int(monitorID)), + ) + + log.Debug("GetHistoryMySQL", "server", serverID, "monitor", monitorID, "since", since, "count", count) q := ntpdb.NewWrappedQuerier(ntpdb.New(db)) @@ -53,6 +83,19 @@ func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, sin return nil, err } + monitors, err := getMonitorNames(ctx, ls, q) + if err != nil { + return nil, err + } + + return &LogScoreHistory{ + LogScores: ls, + Monitors: monitors, + // MonitorIDs: monitorIDs, + }, nil +} + +func getMonitorNames(ctx context.Context, ls []ntpdb.LogScore, q ntpdb.QuerierTx) (map[int]string, error) { monitors := map[int]string{} monitorIDs := []uint32{} for _, l := range ls { @@ -73,10 +116,5 @@ func GetHistory(ctx context.Context, db *sql.DB, serverID, monitorID uint32, sin for _, m := range dbmons { monitors[int(m.ID)] = m.DisplayName() } - - return &LogScoreHistory{ - LogScores: ls, - Monitors: monitors, - MonitorIDs: monitorIDs, - }, nil + return monitors, nil } diff --git a/server/history.go b/server/history.go index 88b166d..2c6e794 100644 --- a/server/history.go +++ b/server/history.go @@ -42,8 +42,17 @@ func paramHistoryMode(s string) historyMode { } } -func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb.Server) (*logscores.LogScoreHistory, error) { - log := logger.Setup() +type historyParameters struct { + limit int + monitorID int + server ntpdb.Server + since time.Time +} + +func (srv *Server) getHistoryParameters(ctx context.Context, c echo.Context) (historyParameters, error) { + log := logger.FromContext(ctx) + + p := historyParameters{} limit := 0 if limitParam, err := strconv.Atoi(c.QueryParam("limit")); err == nil { @@ -51,20 +60,17 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb. } else { limit = 100 } + if limit > 10000 { limit = 10000 } - since, _ := strconv.ParseInt(c.QueryParam("since"), 10, 64) // defaults to 0 so don't care if it parses - - monitorParam := c.QueryParam("monitor") - - if since > 0 { - c.Request().Header.Set("Cache-Control", "s-maxage=300") - } + p.limit = limit q := ntpdb.NewWrappedQuerier(ntpdb.New(srv.db)) + monitorParam := c.QueryParam("monitor") + var monitorID uint32 = 0 switch monitorParam { case "": @@ -84,7 +90,7 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb. // only accept the name prefix; no wildcards; trust the database // to filter out any other crazy if strings.ContainsAny(monitorParam, "_%. \t\n") { - return nil, echo.NewHTTPError(http.StatusNotFound, "monitor not found") + return p, echo.NewHTTPError(http.StatusNotFound, "monitor not found") } if err != nil { @@ -92,22 +98,33 @@ func (srv *Server) getHistory(ctx context.Context, c echo.Context, server ntpdb. monitor, err := q.GetMonitorByName(ctx, sql.NullString{Valid: true, String: monitorParam}) if err != nil { log.Warn("could not find monitor", "name", monitorParam, "err", err) - return nil, echo.NewHTTPError(http.StatusNotFound, "monitor not found") + return p, echo.NewHTTPError(http.StatusNotFound, "monitor not found") } monitorID = monitor.ID } } } + p.monitorID = int(monitorID) log.DebugContext(ctx, "monitor param", "monitor", monitorID) - sinceTime := time.Unix(since, 0) + since, _ := strconv.ParseInt(c.QueryParam("since"), 10, 64) // defaults to 0 so don't care if it parses if since > 0 { - log.Warn("monitor data requested with since parameter, not supported", "since", sinceTime) + p.since = time.Unix(since, 0) + } + if !p.since.IsZero() { + log.Warn("monitor data requested with since parameter", "since", p.since) } - ls, err := logscores.GetHistory(ctx, srv.db, server.ID, monitorID, sinceTime, limit) + return p, nil +} +func (srv *Server) getHistoryCH(ctx context.Context, c echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) { + return logscores.GetHistoryClickHouse(ctx, srv.ch, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit) +} + +func (srv *Server) getHistoryMySQL(ctx context.Context, c echo.Context, p historyParameters) (*logscores.LogScoreHistory, error) { + ls, err := logscores.GetHistoryMySQL(ctx, srv.db, p.server.ID, uint32(p.monitorID), p.since, p.limit) return ls, err } @@ -124,6 +141,13 @@ func (srv *Server) history(c echo.Context) error { return echo.NewHTTPError(http.StatusNotFound, "invalid mode") } + p, err := srv.getHistoryParameters(ctx, c) + if err != nil { + log.Error("get history parameters", "err", err) + span.RecordError(err) + return echo.NewHTTPError(http.StatusInternalServerError, "internal error") + } + server, err := srv.FindServer(ctx, c.Param("server")) if err != nil { log.Error("find server", "err", err) @@ -139,7 +163,15 @@ func (srv *Server) history(c echo.Context) error { return echo.NewHTTPError(http.StatusNotFound, "server not found") } - history, err := srv.getHistory(ctx, c, server) + p.server = server + + var history *logscores.LogScoreHistory + + if c.QueryParam("source") == "c" { + history, err = srv.getHistoryCH(ctx, c, p) + } else { + history, err = srv.getHistoryMySQL(ctx, c, p) + } if err != nil { var httpError *echo.HTTPError if errors.As(err, &httpError) { @@ -202,10 +234,15 @@ func (srv *Server) historyJSON(ctx context.Context, c echo.Context, server ntpdb // log.InfoContext(ctx, "monitor id list", "ids", history.MonitorIDs) + monitorIDs := []uint32{} + for k := range history.Monitors { + monitorIDs = append(monitorIDs, uint32(k)) + } + q := ntpdb.NewWrappedQuerier(ntpdb.New(srv.db)) logScoreMonitors, err := q.GetServerScores(ctx, ntpdb.GetServerScoresParams{ - MonitorIDs: history.MonitorIDs, + MonitorIDs: monitorIDs, ServerID: server.ID, }, ) diff --git a/server/server.go b/server/server.go index 303cc8d..81ce1f7 100644 --- a/server/server.go +++ b/server/server.go @@ -174,6 +174,7 @@ func (srv *Server) Run() error { e.Use(middleware.RecoverWithConfig(middleware.RecoverConfig{ LogErrorFunc: func(c echo.Context, err error, stack []byte) error { log.ErrorContext(c.Request().Context(), err.Error(), "stack", string(stack)) + fmt.Println(string(stack)) return err }, }))