インシデントレスポンス — 顧客の本番環境で火を消す

# ソウタは Nextera の構成を頭に入れていた
# config/customers/nextera_financial.yml に相当する知識
nextera_config = {
  api_version: "v2.3",
  model: "risk-analysis-v4",
  max_concurrent_requests: 500,
  timeout_seconds: 10,
  custom_preprocessing: true,  # 独自の前処理パイプライン
  region: "ap-northeast-1",
  dedicated_inference_pool: "nextera-pool-01"
}

# app/models/incident.rb
class Incident < ApplicationRecord
  enum :severity, {
    sev1: 1, # 全面停止
    sev2: 2, # 主要機能停止
    sev3: 3, # 軽微な障害
    sev4: 4  # 外観・軽微
  }
 
  enum :status, {
    detected: 0,
    acknowledged: 1,
    investigating: 2,
    identified: 3,
    mitigating: 4,
    resolved: 5,
    postmortem: 6
  }
 
  belongs_to :customer
  belongs_to :incident_commander, class_name: "User"
  has_many :timeline_entries, dependent: :destroy
  has_many :action_items, dependent: :destroy
 
  validates :title, :severity, :customer, presence: true
 
  def sla_deadline
    base = detected_at || created_at
    case severity
    when "sev1" then base + 15.minutes
    when "sev2" then base + 30.minutes
    when "sev3" then base + 4.hours
    when "sev4" then base + 24.hours
    end
  end
 
  def sla_breached?
    acknowledged_at.nil? && Time.current > sla_deadline
  end
end

# app/services/escalation_evaluator.rb
class EscalationEvaluator
  TIME_LIMITS = { sev1: 30.minutes, sev2: 2.hours, sev3: 8.hours }.freeze
 
  def initialize(incident)
    @incident = incident
  end
 
  def should_escalate?
    time_exceeded? || error_rate_exceeded? ||
      affected_users_exceeded? || vip_customer_requested?
  end
 
  def escalation_reason
    reasons = []
    reasons << "SLA時間超過" if time_exceeded?
    reasons << "エラー率閾値超過" if error_rate_exceeded?
    reasons << "影響ユーザー数超過" if affected_users_exceeded?
    reasons << "VIP顧客からの要求" if vip_customer_requested?
    reasons.join(", ")
  end
 
  private
 
  def time_exceeded?
    limit = TIME_LIMITS[@incident.severity.to_sym]
    limit && !@incident.resolved? && (Time.current - @incident.detected_at > limit)
  end
 
  def error_rate_exceeded?
    @incident.customer.current_error_rate_percent > 50.0
  end
 
  def affected_users_exceeded?
    @incident.affected_user_count > 1000
  end
 
  def vip_customer_requested?
    @incident.customer.vip? && @incident.customer_escalation_requested?
  end
end

# app/models/incident_role.rb
class IncidentRole < ApplicationRecord
  enum :role_type, {
    incident_commander: 0,  # 全体統括
    delta_lead: 1,          # 技術チームリード
    delta_member: 2,        # 技術調査メンバー
    echo_lead: 3,           # コミュニケーションリード
    echo_member: 4,         # コミュニケーション担当
    fde_bridge: 5           # FDE（Delta/Echo 橋渡し）
  }
 
  belongs_to :incident
  belongs_to :user
 
  def can_update_customer?
    echo_lead? || echo_member? || fde_bridge?
  end
 
  def can_deploy_fix?
    delta_lead? || delta_member? || fde_bridge?
  end
end

# bin/incident_reproduce.rb — 再現テストスクリプト
customer = Customer.find_by!(slug: "nextera-financial")
api_client = ArclightApi::Client.new(customer:, environment: :production)
 
10.times do |i|
  start = Time.current
  begin
    api_client.predict(model: "risk-analysis-v4", input: sample_risk_payload)
    puts "[#{i + 1}] 成功 - #{(Time.current - start).round(2)}s"
  rescue ArclightApi::TimeoutError, ArclightApi::ServerError => e
    puts "[#{i + 1}] 失敗 - #{(Time.current - start).round(2)}s - #{e.class}"
  end
end
# => 10回中6回タイムアウト、2回サーバーエラー。再現率80%

# 他の顧客の API レスポンスタイムを確認
aws cloudwatch get-metric-statistics \
  --namespace "ArclightAI/API" \
  --metric-name "ResponseTime" \
  --dimensions Name=Customer,Value=nextera-financial \
  --start-time "2026-06-28T16:00:00Z" \
  --end-time "2026-06-28T17:15:00Z" \
  --period 300 \
  --statistics Average Maximum \
  --region ap-northeast-1

# app/services/inference_pool_diagnostics.rb
class InferencePoolDiagnostics
  def initialize(pool_id)
    @pool_id = pool_id
    @cw = Aws::CloudWatch::Client.new(region: "ap-northeast-1")
  end
 
  def check_model_cache_hit_rate
    latest = fetch_metric("ModelCacheHitRate", ["Average"])
    { cache_hit_rate: latest&.average&.round(2), healthy: latest&.average.to_f > 80.0 }
  end
 
  def check_gpu_memory_usage
    latest = fetch_metric("GPUMemoryUsage", ["Average", "Maximum"])
    { avg_usage: latest&.average&.round(2), max_usage: latest&.maximum&.round(2) }
  end
 
  private
 
  def fetch_metric(name, stats)
    resp = @cw.get_metric_statistics(
      namespace: "ArclightAI/Inference", metric_name: name,
      dimensions: [{ name: "PoolId", value: @pool_id }],
      start_time: 2.hours.ago, end_time: Time.current,
      period: 300, statistics: stats
    )
    resp.datapoints.max_by(&:timestamp)
  end
end
 
# 実行結果
diag = InferencePoolDiagnostics.new("nextera-pool-01")
diag.check_model_cache_hit_rate  # => { cache_hit_rate: 12.3, healthy: false }
diag.check_gpu_memory_usage      # => { avg_usage: 97.8, max_usage: 99.2 }

# 緩和策: 推論プールの GPU メモリ上限を一時的に拡大
# bin/incident_mitigate.rb
 
# Step 1: 推論プールのスケールアウト
ecs_client = Aws::ECS::Client.new(region: "ap-northeast-1")
ecs_client.update_service(
  cluster: "arclight-inference",
  service: "nextera-pool-01",
  desired_count: 4  # 2 → 4 にスケールアウト
)
 
# Step 2: カスタム前処理のメモリ制限を設定
config_client = ArclightConfig::Client.new
config_client.update_pool_config(
  pool_id: "nextera-pool-01",
  preprocessing_memory_limit_mb: 2048  # 制限なし → 2GB に制限
)
 
# Step 3: モデルキャッシュのウォームアップ
warmup_client = ArclightApi::WarmupClient.new
warmup_client.warmup_model(
  pool_id: "nextera-pool-01",
  model: "risk-analysis-v4"
)
 
puts "緩和策を適用しました。メトリクスの回復を監視します。"

# app/services/postmortem_builder.rb
class PostmortemBuilder
  def initialize(incident)
    @incident = incident
  end
 
  def generate
    {
      title: @incident.title,
      severity: @incident.severity,
      duration: format_duration,
      timeline: build_timeline,
      root_cause: @incident.root_cause_description,
      five_whys: @incident.five_whys_entries.order(:level).map { |e|
        { level: e.level, question: e.question, answer: e.answer }
      },
      action_items: @incident.action_items.map { |item|
        { action: item.description, owner: item.assignee.name,
          priority: item.priority, deadline: item.due_date }
      }
    }
  end
 
  private
 
  def build_timeline
    @incident.timeline_entries.order(:occurred_at).map do |entry|
      { time: entry.occurred_at.strftime("%H:%M JST"),
        event: entry.description, actor: entry.user&.name || "System" }
    end
  end
 
  def format_duration
    seconds = @incident.resolved_at - @incident.detected_at
    "#{(seconds / 3600).floor}時間#{((seconds % 3600) / 60).floor}分"
  end
end

# app/services/monitoring/cloudwatch_reporter.rb
class Monitoring::CloudwatchReporter
  def initialize
    @client = Aws::CloudWatch::Client.new(region: "ap-northeast-1")
    @namespace = "ArclightAI/CustomerMetrics"
  end
 
  def report_api_latency(customer_slug:, endpoint:, latency_ms:)
    put_metric("APILatency", [
      { name: "Customer", value: customer_slug },
      { name: "Endpoint", value: endpoint }
    ], latency_ms, "Milliseconds")
  end
 
  def report_error_rate(customer_slug:, error_count:, total_count:)
    rate = total_count.positive? ? (error_count.to_f / total_count * 100) : 0.0
    put_metric("ErrorRate", [
      { name: "Customer", value: customer_slug }
    ], rate, "Percent")
  end
 
  private
 
  def put_metric(metric_name, dimensions, value, unit)
    @client.put_metric_data(
      namespace: @namespace,
      metric_data: [{
        metric_name: metric_name, dimensions: dimensions,
        timestamp: Time.current, value: value, unit: unit
      }]
    )
  end
end

# lib/tasks/monitoring/setup_alarms.rake
namespace :monitoring do
  desc "顧客環境のCloudWatch Alarmsを設定"
  task setup_alarms: :environment do
    cloudwatch = Aws::CloudWatch::Client.new(region: "ap-northeast-1")
    sns_arn = ENV.fetch("PAGERDUTY_SNS_TOPIC_ARN")
 
    Customer.active.find_each do |customer|
      # エラー率アラーム（5分平均が10%超過 × 2回連続でトリガー）
      cloudwatch.put_metric_alarm(
        alarm_name: "#{customer.slug}-error-rate-high",
        namespace: "ArclightAI/CustomerMetrics",
        metric_name: "ErrorRate",
        dimensions: [{ name: "Customer", value: customer.slug }],
        statistic: "Average", period: 300,
        evaluation_periods: 2, threshold: 10.0,
        comparison_operator: "GreaterThanThreshold",
        alarm_actions: [sns_arn]
      )
 
      # P99 レイテンシアラーム（5秒超過 × 3回連続）
      cloudwatch.put_metric_alarm(
        alarm_name: "#{customer.slug}-latency-high",
        namespace: "ArclightAI/CustomerMetrics",
        metric_name: "APILatency",
        dimensions: [{ name: "Customer", value: customer.slug }],
        statistic: "p99", period: 300,
        evaluation_periods: 3, threshold: 5000.0,
        comparison_operator: "GreaterThanThreshold",
        alarm_actions: [sns_arn]
      )
    end
  end
end

# app/services/incident_auto_creator.rb
class IncidentAutoCreator
  def self.from_cloudwatch_alarm(alarm_message)
    parsed = JSON.parse(alarm_message)
    customer = Customer.find_by!(slug: extract_customer_slug(parsed))
 
    incident = Incident.create!(
      title: "#{customer.name}: #{parsed['AlarmDescription']}",
      severity: determine_severity(parsed),
      status: :detected, customer: customer,
      detected_at: Time.current, source: "cloudwatch_alarm"
    )
 
    incident.timeline_entries.create!(
      occurred_at: Time.current,
      description: "CloudWatch Alarm トリガー: #{parsed['AlarmName']}",
      entry_type: :automated
    )
 
    PagerdutyNotifier.trigger(
      severity: incident.severity, summary: incident.title,
      source: "ArclightAI Monitoring", component: customer.slug
    )
    incident
  end
 
  private
 
  def self.determine_severity(data)
    case data.dig("Trigger", "MetricName")
    when "ErrorRate" then data.dig("Trigger", "Threshold") >= 50 ? :sev1 : :sev2
    when "APILatency" then data.dig("Trigger", "Threshold") >= 30_000 ? :sev1 : :sev2
    else :sev3
    end
  end
 
  def self.extract_customer_slug(data)
    data.dig("Trigger", "Dimensions")&.find { |d| d["name"] == "Customer" }&.dig("value")
  end
end