Estratégia de Rollback em Progressive Delivery

# Step 1: Kill the flag immediately
curl -X PATCH "https://flags.internal/api/flags/new-checkout-flow" \
  -H "Authorization: Bearer $FLAG_TOKEN" \
  -d '{"rollout": {"percentage": 0}}'

# Step 2: Verify latency is recovering
# Check p99 over last 5 minutes vs previous hour
curl -s "http://prometheus:9090/api/v1/query" \
  --data-urlencode 'query=
    histogram_quantile(0.99,
      rate(http_request_duration_seconds_bucket{
        service="checkout"
      }[5m])
    )' | jq '.data.result[0].value[1]'

# Step 3: Check for lingering damage
# Queue depth - should be draining, not growing
curl -s "http://prometheus:9090/api/v1/query" \
  --data-urlencode 'query=
    rabbitmq_queue_messages{queue="checkout-events"}' \
  | jq '.data.result[0].value[1]'

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: latency-check
spec:
  metrics:
  - name: p99-latency
    interval: 1m
    count: 5
    failureLimit: 2         # 2 failures out of 5 checks = auto rollback
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          histogram_quantile(0.99,
            rate(http_request_duration_seconds_bucket{
              service="checkout",
              version="canary"
            }[2m])
          )
    # Fail if canary p99 is more than 3x the stable version
    successCondition: "result[0] < 0.9"
    failureCondition: "result[0] >= 0.9"

# progressive_delivery/auto_rollback.py
import requests
from prometheus_client import CollectorRegistry

def check_and_rollback(flag_key, metric_query, threshold, flag_service_url):
    """Auto-disable flag if metric exceeds threshold."""
    result = requests.get(
        "http://prometheus:9090/api/v1/query",
        params={"query": metric_query}
    ).json()

    current_value = float(result["data"]["result"][0]["value"][1])

    if current_value > threshold:
        # Kill the flag
        requests.patch(
            f"{flag_service_url}/api/flags/{flag_key}",
            json={"rollout": {"percentage": 0}},
            headers={"Authorization": f"Bearer {FLAG_TOKEN}"}
        )
        # Alert the team
        requests.post(SLACK_WEBHOOK, json={
            "text": f":rotating_light: Auto-rolled back flag `{flag_key}`. "
                    f"Metric breached threshold: {current_value:.2f} > {threshold}"
        })
        return False
    return True