- alert: HighErrorRate
  expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: 'High error rate detected'

- alert: HighLatency
  expr: histogram_quantile(0.95, http_request_duration_seconds_bucket) > 1
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: 'High latency detected'

- alert: ServiceDown
  expr: up{job=~".*-service"} == 0
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: 'Service is down'

- alert: DatabaseConnectionPoolHigh
  expr: pg_stat_activity_count / pg_settings_max_connections > 0.8
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: 'Database connection pool high'

- alert: KafkaConsumerLagHigh
  expr: kafka_consumer_lag > 1000
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: 'Kafka consumer lag is high'

import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { JaegerExporter } from '@opentelemetry/exporter-jaeger';

const provider = new NodeTracerProvider();
const exporter = new JaegerExporter({
  endpoint: 'http://jaeger:14268/api/traces',
});

provider.addSpanProcessor(new SimpleSpanProcessor(exporter));
provider.register();

import axios from 'axios';

export async function sendSlackAlert(message: string, severity: 'info' | 'warning' | 'critical') {
  const webhookUrl = process.env.SLACK_WEBHOOK_URL;

  const colors = {
    info: '#36a64f',
    warning: '#ff9900',
    critical: '#ff0000',
  };

  await axios.post(webhookUrl, {
    attachments: [
      {
        color: colors[severity],
        text: message,
        ts: Math.floor(Date.now() / 1000),
      },
    ],
  });
}

export async function sendPagerDutyAlert(
  summary: string,
  severity: 'critical' | 'error' | 'warning',
) {
  const apiKey = process.env.PAGERDUTY_API_KEY;

  await axios.post('https://events.pagerduty.com/v2/enqueue', {
    routing_key: apiKey,
    event_action: 'trigger',
    payload: {
      summary,
      severity,
      source: 'Maniton Platform',
      custom_details: {
        timestamp: new Date().toISOString(),
      },
    },
  });
}

# Проверка endpoint
curl http://localhost:3001/metrics

# Проверка Prometheus конфигурации
kubectl get configmap prometheus -o yaml

# Перезапуск Prometheus
kubectl rollout restart deployment prometheus

# Проверка Alertmanager
curl http://localhost:9093/-/healthy

# Проверка webhook
curl -X POST https://hooks.slack.com/services/... -d '{"text":"test"}'

# Проверка конфигурации
kubectl get configmap alertmanager -o yaml

# Проверка Loki
curl http://localhost:3100/ready

# Поиск логов
curl -G 'http://localhost:3100/loki/api/v1/query_range' \
  --data-urlencode 'query={job="auth-service"}' \
  --data-urlencode 'start=2024-01-01T00:00:00Z' \
  --data-urlencode 'end=2024-01-01T23:59:59Z'