diff --git a/README.md b/README.md index 352b7ee..672c853 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,27 @@ -```bash -mkdir -p ./v/grafana +cloud vm +```bash sudo ufw allow in on br-monitor sudo ufw allow out on br-monitor +mkdir -p ./v/grafana + +docker compose up +``` + +local + +```bash +docker run --rm -it -p 6379:6379 redis:8.0.3-alpine3.21 + autossh -M 20002 -N \ -L 127.0.0.1:3000:127.0.0.1:3000 \ -L 127.0.0.1:6060:127.0.0.1:6060 \ -R 10.0.10.1:6379:127.0.0.1:6379 \ gcp-tunnel -curl 'http://localhost:6060/api/v1/metrics?service=redis&t0=0&t1=1798732800' | jq +cd terraform +terraform apply + +curl 'http://localhost:6060/api/v1/metrics?service=redis&t0=0&t1=32503651200' | jq ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 562951d..35d48df 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,9 +34,3 @@ networks: config: - subnet: 10.0.10.0/24 gateway: 10.0.10.1 - - # redis: - # image: redis:8.0.3-alpine3.21 - # ports: - # - 127.0.0.1:6379:6379 - # restart: unless-stopped diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..33f5e4c --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,5 @@ +.terraform +.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..f264c00 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,126 @@ +terraform { + required_providers { + grafana = { + source = "grafana/grafana" + version = "4.27.0" + } + } +} + +provider "grafana" { + url = var.grafana_url + auth = var.grafana_auth +} + +resource "grafana_data_source" "api" { + name = "api" + type = "yesoreyeram-infinity-datasource" + json_data_encoded = jsonencode({ + datasource_mode = "basic" + }) +} + +resource "grafana_dashboard" "uptime" { + config_json = jsonencode({ + title = "uptime" + uid = "uptime_dashboard" + panels = [{ + id = 1 + type = "timeseries" + title = "redis" + gridPos = { h = 10, w = 16 } + datasource = { uid = grafana_data_source.api.uid } + targets = [{ + type = "json" + source = "url" + url = "http://datasource:6060/api/v1/metrics" + url_options = { + params = [ + { key = "service", value = "redis" }, + { key = "t0", value = "$${__from:date:seconds}" }, + { key = "t1", value = "$${__to:date:seconds}" } + ] + } + columns = [ + { selector = "timestamp_sec", text = "time", type = "timestamp_epoch_s" }, + { selector = "up", type = "number" } + ] + }] + }] + }) +} + +resource "grafana_folder" "rules" { + title = "rules" +} + +resource "grafana_rule_group" "alert_rule" { + name = "alert rule" + folder_uid = grafana_folder.rules.uid + interval_seconds = 10 + rule { + name = "down" + condition = "B" + for = "30s" + annotations = { + "__dashboardUid__" = grafana_dashboard.uptime.uid + "__panelId__" = "1" + "description" = "redis has been down for over 30 seconds" + } + data { + ref_id = "A" + datasource_uid = grafana_data_source.api.uid + model = jsonencode({ + type = "json" + source = "url" + parser = "backend" + format = "timeseries" + url = "http://datasource:6060/api/v1/metrics" + url_options = { + params = [ + { key = "service", value = "redis" }, + { key = "t0", value = "$${__from:date:seconds}" }, + { key = "t1", value = "$${__to:date:seconds}" } + ] + } + columns = [ + { selector = "timestamp_sec", text = "time", type = "timestamp_epoch_s" }, + { selector = "up", type = "number" } + ] + }) + relative_time_range { + from = 60 + to = 0 + } + } + data { + ref_id = "B" + datasource_uid = "__expr__" + relative_time_range { + from = 0 + to = 0 + } + model = jsonencode({ + type = "classic_conditions" + conditions = [{ + evaluator = { params = [1], type = "lt" }, + reducer = { type = "last" }, + query = { params = ["A"] } + }] + }) + } + } +} + +resource "grafana_contact_point" "telegram" { + name = "telegram" + telegram { + token = var.telegram_bot_token + chat_id = var.telegram_chat_id + } +} + +resource "grafana_notification_policy" "main" { + group_by = ["..."] + contact_point = grafana_contact_point.telegram.name +} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100644 index 0000000..a5d902d --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,4 @@ +grafana_auth = "pika:boingboing" + +telegram_bot_token = "0000000000:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +telegram_chat_id = "-1009999999999" diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..b9564d3 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,19 @@ +variable "grafana_url" { + type = string + default = "http://localhost:3000" +} + +variable "grafana_auth" { + type = string + sensitive = true +} + +variable "telegram_bot_token" { + type = string + sensitive = true +} + +variable "telegram_chat_id" { + type = string + sensitive = true +}