diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..73873ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.vscode +*.db +*.duckdb +v diff --git a/README.md b/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/datasource/Dockerfile b/datasource/Dockerfile new file mode 100644 index 0000000..623f3c7 --- /dev/null +++ b/datasource/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.25.7-trixie AS build +ENV CGO_ENABLED=1 +WORKDIR /app +COPY go.mod go.sum . +RUN go mod download +COPY . . +RUN go build -o main . + +FROM ubuntu:24.04 +COPY --from=build /app/main /app/main +WORKDIR /app +CMD ["./main"] diff --git a/datasource/collector/collector.go b/datasource/collector/collector.go new file mode 100644 index 0000000..f04a8e9 --- /dev/null +++ b/datasource/collector/collector.go @@ -0,0 +1,9 @@ +package collector + +import ( + "context" +) + +type Collector interface { + Collect(context.Context) error +} diff --git a/datasource/collector/redis.go b/datasource/collector/redis.go new file mode 100644 index 0000000..ac8863c --- /dev/null +++ b/datasource/collector/redis.go @@ -0,0 +1,77 @@ +package collector + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "datasource/config" + "datasource/store" + "datasource/types" +) + +type RedisCollector struct { + rdb *redis.Client + store *store.Store +} + +func NewRedisCollector(cfg *config.RedisConfig, store *store.Store) *RedisCollector { + rdb := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + Password: cfg.Password, + }) + return &RedisCollector{rdb: rdb, store: store} +} + +func (c *RedisCollector) Close() error { + return c.rdb.Close() +} + +func (c *RedisCollector) Collect(ctx context.Context) error { + metric := new(types.Metric) + + info, err := c.rdb.Info(ctx, "server", "memory").Result() + + if err != nil { + // todo (david): log but don't return the error as we still need to save the metric + } else { + metric.Up = 1 + + for _, line := range strings.Split(info, "\r\n") { + kv := strings.Split(line, ":") + if len(kv) < 2 { + continue + } + + k, v := kv[0], kv[1] + + switch k { + case "uptime_in_seconds": + if uptimeSec, err := strconv.ParseInt(v, 10, 64); err != nil { + return err + } else { + metric.UptimeSec = uptimeSec + } + case "used_memory": + if memoryUsageBytes, err := strconv.ParseInt(v, 10, 64); err != nil { + return err + } else { + metric.MemoryUsageBytes = memoryUsageBytes + } + } + } + } + + metric.Service = "redis" + metric.TimestampSec = time.Now().Unix() + + if err := c.store.SaveMetric(ctx, metric); err != nil { + return err + } + + return nil +} diff --git a/datasource/config.ini b/datasource/config.ini new file mode 100644 index 0000000..c05f52b --- /dev/null +++ b/datasource/config.ini @@ -0,0 +1,8 @@ +[data_source] +port=6060 +db_path=./data/metrics.duckdb + +[redis] +host=127.0.0.1 +port=6379 +; password=boingboing diff --git a/datasource/config/config.go b/datasource/config/config.go new file mode 100644 index 0000000..462e037 --- /dev/null +++ b/datasource/config/config.go @@ -0,0 +1,88 @@ +package config + +import ( + "fmt" + "strings" + + "gopkg.in/ini.v1" +) + +type DataSourceConfig struct { + Port int + DBPath string +} + +type RedisConfig struct { + Host string + Port int + Password string +} + +func loadRequiredInt(sec *ini.Section, key string) (int, error) { + val, err := sec.GetKey(key) + if err != nil { + return 0, err + } + res, err := val.Int() + if err != nil { + return 0, err + } + return res, nil +} + +func loadRequiredString(sec *ini.Section, key string) (string, error) { + val, err := sec.GetKey(key) + if err != nil { + return "", err + } + res := strings.TrimSpace(val.String()) + if res == "" { + return "", fmt.Errorf("%s cannot be empty", key) + } + return res, nil +} + +func LoadDataSourceConfig(cfg *ini.File) (*DataSourceConfig, error) { + sec, err := cfg.GetSection("data_source") + if err != nil { + return nil, err + } + + port, err := loadRequiredInt(sec, "port") + if err != nil { + return nil, err + } + + dbPath, err := loadRequiredString(sec, "db_path") + if err != nil { + return nil, err + } + + return &DataSourceConfig{ + Port: port, + DBPath: dbPath, + }, nil +} + +func LoadRedisConfig(cfg *ini.File) (*RedisConfig, error) { + sec, err := cfg.GetSection("redis") + if err != nil { + return nil, err + } + + host, err := loadRequiredString(sec, "host") + if err != nil { + return nil, err + } + + port, err := loadRequiredInt(sec, "port") + if err != nil { + return nil, err + } + + return &RedisConfig{ + Host: host, + Port: port, + Password: strings.TrimSpace(sec.Key("password").String()), + }, nil +} diff --git a/datasource/controller/controller.go b/datasource/controller/controller.go new file mode 100644 index 0000000..800af76 --- /dev/null +++ b/datasource/controller/controller.go @@ -0,0 +1,71 @@ +package controller + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strconv" + + "datasource/store" +) + +type Controller struct { + store *store.Store +} + +func NewController(store *store.Store) *Controller { + return &Controller{store: store} +} + +func (c *Controller) GetMetrics(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + + service := query.Get("service") + if service == "" { + http.Error(w, "service cannot be empty", http.StatusBadRequest) + return + } + + t0, err := strconv.ParseInt(query.Get("t0"), 10, 64) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + t1, err := strconv.ParseInt(query.Get("t1"), 10, 64) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if t0 > t1 { + http.Error(w, "t0 cannot be larger than t1", http.StatusBadRequest) + return + } + + metrics, err := c.store.GetMetrics(r.Context(), service, t0, t1) + if err != nil { + // todo (david): log internal server error + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(metrics); err != nil { + // todo (david): log internal server error + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + + if _, err := io.Copy(w, &buf); err != nil { + // too late to return http.StatusInternalServerError + // todo (david): log internal server error + } +} + +// todo (david) +// func (c *Controller) DeleteMetrics(w http.ResponseWriter, r *http.Request) {} diff --git a/datasource/go.mod b/datasource/go.mod new file mode 100644 index 0000000..b790298 --- /dev/null +++ b/datasource/go.mod @@ -0,0 +1,36 @@ +module datasource + +go 1.24.2 + +require ( + github.com/duckdb/duckdb-go/v2 v2.5.5 + github.com/redis/go-redis/v9 v9.17.3 + gopkg.in/ini.v1 v1.67.1 +) + +require ( + github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/duckdb/duckdb-go-bindings v0.3.3 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.3.3 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.3.3 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.3.3 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.3.3 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.3.3 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 // indirect + golang.org/x/tools v0.41.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect +) diff --git a/datasource/go.sum b/datasource/go.sum new file mode 100644 index 0000000..055bc77 --- /dev/null +++ b/datasource/go.sum @@ -0,0 +1,96 @@ +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/duckdb/duckdb-go-bindings v0.3.3 h1:lXogtCY8hiGLQvTfK55HcgvaA3K2MrwKeZGqhIin35U= +github.com/duckdb/duckdb-go-bindings v0.3.3/go.mod h1:zS7OpBP8zwVlP38OljRZOnqWYlNd4KLcVfMoA1JFzpk= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.3.3 h1:ue8BtIOSt+2Bt2fEfTAvBcQLxzBFhgfCcyzPtqQWTRA= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.3.3/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.3.3 h1:2TrSeTgtwi3WIvub9ba0mny+AClSNo1w0Ghszc2B8lQ= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.3.3/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.3.3 h1:GN0cexhfE7uLb7qgDmsYG324wKF15nW+O7v5+NGalS4= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.3.3/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.3.3 h1:bIJV+ct6yvMXjy+N3bfILFd0fkTK50AUhUTerkY40/8= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.3.3/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.3.3 h1:SK2sunA/MPb2T3113iFzHv6DWeu+qrsw0DizTFrvM+Q= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.3.3/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.5.5 h1:TlK8ipnzoKW2aNrjGqRkFWLCDpJDxR/VwH8ezEcvVhw= +github.com/duckdb/duckdb-go/v2 v2.5.5/go.mod h1:6uIbC3gz36NCEygECzboygOo/Z9TeVwox/puG+ohWV0= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1DQx4= +github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 h1:i0p03B68+xC1kD2QUO8JzDTPXCzhN56OLJ+IhHY8U3A= +golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= +gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/datasource/main.go b/datasource/main.go new file mode 100644 index 0000000..5d39de4 --- /dev/null +++ b/datasource/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + _ "github.com/duckdb/duckdb-go/v2" + "gopkg.in/ini.v1" + + "datasource/collector" + "datasource/config" + "datasource/controller" + "datasource/store" +) + +func main() { + // todo (david): init logger + + var wg sync.WaitGroup + defer wg.Wait() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + cfg, err := ini.Load("./config.ini") + if err != nil { + log.Fatal(err) + } + + dataSourceConfig, err := config.LoadDataSourceConfig(cfg) + if err != nil { + log.Fatal(err) + } + + store, err := store.NewStore(dataSourceConfig.DBPath) + if err != nil { + log.Fatal(err) + } + defer store.Close() + + redisConfig, err := config.LoadRedisConfig(cfg) + if err != nil { + log.Fatal(err) + } + + redisCollector := collector.NewRedisCollector(redisConfig, store) + defer redisCollector.Close() + + collectors := []collector.Collector{redisCollector} + + for _, collector := range collectors { + wg.Add(1) + go func() { + defer wg.Done() + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := collector.Collect(ctx); err != nil { + log.Printf("failed to collect: %v", err) + } + } + } + }() + } + + controller := controller.NewController(store) + + mux := http.NewServeMux() + mux.HandleFunc("GET /api/v1/metrics", controller.GetMetrics) + // todo (david) + // mux.HandleFunc("DELETE /api/v1/metrics", controller.DeleteMetrics) + + srv := &http.Server{ + Addr: fmt.Sprintf("0.0.0.0:%d", dataSourceConfig.Port), + Handler: mux, + } + + wg.Add(1) + go func() { + defer wg.Done() + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("http.ListenAndServe: %v\n", err) + } + }() + + <-ctx.Done() + + srv.Shutdown(context.Background()) + + srvShutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(srvShutdownCtx); err != nil { + log.Fatalf("srv.Shutdown: %v", err) + } +} diff --git a/datasource/store/duckdb.go b/datasource/store/duckdb.go new file mode 100644 index 0000000..cea117e --- /dev/null +++ b/datasource/store/duckdb.go @@ -0,0 +1,124 @@ +package store + +import ( + "context" + "database/sql" + + "datasource/types" +) + +type Store struct { + db *sql.DB +} + +func NewStore(dbPath string) (*Store, error) { + db, err := sql.Open("duckdb", dbPath) + if err != nil { + return nil, err + } + + // keep one connection ready for collector's next tick + db.SetMaxIdleConns(1) + + // force sequential access to prevent DuckDB locking errors + db.SetMaxOpenConns(1) + + migrationQuery := ` + CREATE TABLE IF NOT EXISTS metrics ( + service VARCHAR, + timestamp_sec BIGINT, + up INTEGER, + uptime_sec BIGINT, + memory_usage_bytes BIGINT + )` + + if _, err := db.Exec(migrationQuery); err != nil { + return nil, err + } + + return &Store{db: db}, nil +} + +func (s *Store) Close() error { + return s.db.Close() +} + +func (s *Store) SaveMetric(ctx context.Context, metric *types.Metric) error { + query := ` + INSERT INTO metrics ( + service, + timestamp_sec, + up, + uptime_sec, + memory_usage_bytes + ) VALUES (?, ?, ?, ?, ?)` + + _, err := s.db.ExecContext(ctx, query, + metric.Service, + metric.TimestampSec, + metric.Up, + metric.UptimeSec, + metric.MemoryUsageBytes, + ) + + return err +} + +func (s *Store) GetMetrics(ctx context.Context, service string, t0, t1 int64) ([]*types.Metric, error) { + query := ` + SELECT + timestamp_sec, + up, + uptime_sec, + memory_usage_bytes + FROM metrics + WHERE + service = ? AND + timestamp_sec BETWEEN ? AND ? + ORDER BY timestamp_sec` + + rows, err := s.db.QueryContext(ctx, query, service, t0, t1) + if err != nil { + return nil, err + } + defer rows.Close() + + var metrics []*types.Metric + + for rows.Next() { + metric := &types.Metric{Service: service} + if err := rows.Scan(&metric.TimestampSec, &metric.Up, &metric.UptimeSec, &metric.MemoryUsageBytes); err != nil { + return nil, err + } + metrics = append(metrics, metric) + } + + return metrics, nil +} + +func (s *Store) GetAllMetrics(ctx context.Context) ([]*types.Metric, error) { + query := "SELECT service, timestamp_sec, up, uptime_sec, memory_usage_bytes FROM metrics" + + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + + var metrics []*types.Metric + + for rows.Next() { + metric := new(types.Metric) + if err := rows.Scan(&metric.Service, &metric.TimestampSec, &metric.Up, &metric.UptimeSec, &metric.MemoryUsageBytes); err != nil { + return nil, err + } + metrics = append(metrics, metric) + } + + return metrics, nil +} + +// todo (david) +// func (s *Store) DeleteMetrics(ctx context.Context, t0, t1 int64) error { +// return nil +// } diff --git a/datasource/types/types.go b/datasource/types/types.go new file mode 100644 index 0000000..df2a594 --- /dev/null +++ b/datasource/types/types.go @@ -0,0 +1,9 @@ +package types + +type Metric struct { + Service string `json:"service"` + TimestampSec int64 `json:"timestamp_sec"` // unix epoch + Up int `json:"up"` // 0: down, 1: healthy + UptimeSec int64 `json:"uptime_sec"` + MemoryUsageBytes int64 `json:"memory_usage_bytes"` +} diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..b5487b8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,26 @@ +services: + datasource: + build: ./datasource + network_mode: host + volumes: + - ./datasource/config.ini:/app/config.ini:ro + - ./v/datasource/data:/app/data + restart: unless-stopped + + grafana: + image: grafana/grafana:12.3.2 + user: 1000:1000 + environment: + - GF_PLUGINS_PREINSTALL_SYNC=yesoreyeram-infinity-datasource@3.7.0 + - GF_SECURITY_ADMIN_USER=pika + - GF_SECURITY_ADMIN_PASSWORD=boingboing + network_mode: host + volumes: + - ./v/grafana:/var/lib/grafana + restart: unless-stopped + + # redis: + # image: redis:8.0.3-alpine3.21 + # ports: + # - 127.0.0.1:6379:6379 + # restart: unless-stopped