Apache Doris uses the Prometheus alertmanager module to send exception information to the nail alarm group

* groupadd prometheus* useradd -g prometheus -M -s /sbin/nologin prometheus

* wget https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz* tar xf alertmanager-0.23.0.linux-amd64.tar.gz -C /soft* cd /soft* mv alertmanager-0.23.0.linux-amd64 alertmanager* cd alertmanager* mkdir data # System file persistence path * chown -R prometheus.prometheus /soft/alertmanager

route:  group_by: ['alertname']  group_wait: 1s  group_interval: 1m  repeat_interval: 4h  receiver: 'webhook2'receivers:- name: 'webhook2'  webhook_configs:    - &dingtalk_config       send_resolved: true       url: http://localhost:8060/dingtalk/webhook2/send#An inhibition rule mutes an alert (target) matching a set of matchers when an alert (source) exists that matches another set of matchers. Both target and source alerts must have the same label values for the label names in the equal list.inhibit_rules:  - source_match:      severity: 'critical'    target_match:      severity: 'warning'    equal: ['alertname', 'dev', 'instance']

* cd /soft/alertmanager* ./amtool check-config ./alertmanager.yml

* vim /usr/lib/systemd/system/alertmanager.service[Unit]Description=alertmanagerDocumentation=https://prometheus.io/After=network.target[Service]User=prometheusGroup=prometheusExecStart=/soft/alertmanager/alertmanager --config.file=/soft/alertmanager/alertmanager.yml --storage.path=/soft/alertmanager/dataRestart=on-failure[Install]WantedBy=multi-user.target

systemctl daemon-reloadsystemctl enable alertmanager.servicesystemctl start alertmanager.servicesystemctl status alertmanager.service # View service status systemctl restart alertmanager.service # Restart the service to use 

vim prometheus.yml alerting:  alertmanagers:  - static_configs:    - targets: ['localhost:9093']  # According to the fact alertmanager Service address configuration .rule_files:   - "rule/*.yml"   # Custom rule storage directory , Can pass *.yml To configure all rules under this directory .

* ./promtool check config  prometheus.yml* systemctl reload prometheus.service

* wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v1.4.0/prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz* tar -xf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz  -C  /soft* mv prometheus-webhook-dingtalk-1.4.0.linux-amd64 prometheus-webhook-dingtalk

* cp  config.example.yml  config.yml## Request timeout# timeout: 5s## Customizable templates path    Custom template location templates:  - /soft/alertmanager/alarm_template/webhook.tmpl## You can also override default template using `default_message`## The following example to use the 'legacy' template from v0.3.0# default_message:#   title: '{{ template "legacy.title" . }}'#   text: '{{ template "legacy.content" . }}'## Targets, previously was known as "profiles"targets:  webhook1:   # Countersigned nail , You need to configure the key information of the signing robot at the same time     url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx    # secret for signature    secret: SEC000000000000000000000  webhook2:  # No sign and nail     url: https://oapi.dingtalk.com/robot/send?access_token=cf9c2fd69723661108b7fd7******  webhook_legacy:       url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx    # Customize template content    message:      # Use legacy template      title: '{{ template "legacy.title" . }}'      text: '{{ template "legacy.content" . }}'  webhook_mention_all:  #@ Everyone nails     url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx    mention:      all: true  webhook_mention_users:  #@ Specify user nails     url: https://oapi.dingtalk.com/robot/send?access_token=cf9c2fd69723661108b7fd7****    mention:      mobiles: ['152****30', '134****74']

vim /usr/lib/systemd/system/webhook-dingtalk.service[Unit][Unit]Description=prometheus-webhook-dingtalkDocumentation=https://github.com/timonwong/prometheus-webhook-dingtalkAfter=network.target[Service]User=prometheusGroup=prometheusExecStart=/soft/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk  --config.file=/soft/prometheus-webhook-dingtalk/config.ymlRestart=on-failure[Install]WantedBy=multi-user.target

systemctl daemon-reloadsystemctl enable webhook-dingtalk.servicesystemctl start webhook-dingtalk.servicesystemctl status webhook-dingtalk.servicesystemctl restart webhook-dingtalk.service

* vim doris_instance.ymlgroups:  - name: doris_instance_down    rules:      - alert: Doris Backends Down        expr: up {group="be", job="pro-doris"} == 0        for: 20s        labels:          user: doris          severity: error        annotations:          summary: "doris Instance {{ $labels.instance }} down"          description: "doris {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 20s."      - alert: Doris Frontends Down        expr: up {group="fe", job="pro-doris"} == 0        for: 20s        labels:          user: doris          severity: error        annotations:          summary: "doris Instance {{ $labels.instance }} down"          description: "doris {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 20s."

* cd /soft/prometheus* ./promtool check config  prometheus.yml

* systemctl reload prometheus.service