AI 時代の FDE — エージェント、RAG、Eval の最前線

# app/services/document_chunker.rb
class DocumentChunker
  STRATEGIES = { fixed: FixedChunker, semantic: SemanticChunker,
                 recursive: RecursiveChunker }.freeze
 
  def initialize(strategy: :recursive, chunk_size: 512, overlap: 64)
    @strategy = STRATEGIES.fetch(strategy)
    @chunk_size = chunk_size
    @overlap = overlap
  end
 
  def chunk(document)
    @strategy.new(chunk_size: @chunk_size, overlap: @overlap)
             .call(document.content)
  end
end
 
# 再帰的チャンキング -- 構造を保持しながら分割
class RecursiveChunker
  SEPARATORS = ["\n## ", "\n### ", "\n\n", "\n", "。", ""].freeze
 
  def initialize(chunk_size:, overlap:)
    @chunk_size = chunk_size
    @overlap = overlap
  end
 
  def call(text)
    split_recursive(text, SEPARATORS)
  end
 
  private
 
  def split_recursive(text, separators)
    return [text] if text.length <= @chunk_size || separators.empty?
 
    sep = separators.first
    parts = text.split(sep).reject(&:empty?)
    chunks = []
    current = ""
 
    parts.each do |part|
      candidate = current.empty? ? part : "#{current}#{sep}#{part}"
      if candidate.length <= @chunk_size
        current = candidate
      else
        chunks << current unless current.empty?
        current = part
      end
    end
    chunks << current unless current.empty?
 
    chunks.flat_map do |chunk|
      chunk.length > @chunk_size ? split_recursive(chunk, separators[1..]) : chunk
    end
  end
end

# db/migrate/20260601000000_create_document_chunks.rb
class CreateDocumentChunks < ActiveRecord::Migration[8.0]
  def up
    enable_extension "vector"
 
    create_table :document_chunks do |t|
      t.references :document, null: false, foreign_key: true
      t.text :content, null: false
      t.integer :position, null: false
      t.column :embedding, :vector, limit: 1024
      t.jsonb :metadata, default: {}
      t.timestamps
    end
 
    add_index :document_chunks, :embedding,
              using: :hnsw, opclass: :vector_cosine_ops,
              name: "index_document_chunks_on_embedding"
  end
end

# app/models/document_chunk.rb
class DocumentChunk < ApplicationRecord
  belongs_to :document
  has_neighbors :embedding  # pgvector の neighbor gem
 
  scope :search_by_similarity, ->(query_embedding, limit: 5) {
    nearest_neighbors(:embedding, query_embedding, distance: "cosine")
      .limit(limit)
  }
end

# app/services/rag_query_service.rb
class RagQueryService
  MAX_CONTEXT_TOKENS = 4000
 
  def initialize(embedding_service: EmbeddingService.new,
                 llm_client: BedrockLlmClient.new)
    @embedding_service = embedding_service
    @llm_client = llm_client
  end
 
  def query(question, filters: {})
    query_embedding = @embedding_service.embed(question)
    chunks = DocumentChunk.search_by_similarity(query_embedding, limit: 20)
    chunks = chunks.where("metadata @> ?", filters.to_json) if filters.any?
 
    ranked = rerank(question, chunks.first(10))
    context = build_context(ranked.first(5))
    answer = generate(question, context)
 
    Conversation.create!(
      question: question, context_used: context,
      answer: answer, chunk_ids: ranked.map(&:id),
      model_version: @llm_client.model_id
    )
    answer
  end
 
  private
 
  def rerank(question, chunks)
    chunks.sort_by { |c| -compute_relevance_score(question, c.content) }
  end
 
  def build_context(chunks)
    parts, total = [], 0
    chunks.each do |chunk|
      tokens = chunk.content.length / 2  # 日本語の概算
      break if total + tokens > MAX_CONTEXT_TOKENS
      parts << "[出典: #{chunk.document.title}]\n#{chunk.content}"
      total += tokens
    end
    parts.join("\n\n---\n\n")
  end
 
  def generate(question, context)
    @llm_client.generate(
      system: system_prompt,
      messages: [{ role: "user",
                   content: "参照文書:\n#{context}\n\n質問: #{question}" }]
    )
  end
 
  def system_prompt
    <<~PROMPT
      あなたは帝都生命の保険査定アシスタントです。
      1. 参照文書に記載されている情報のみに基づいて回答する
      2. 参照文書にない情報は「該当する情報が見つかりません」と回答する
      3. 金額や日付は参照文書の記載を正確に引用する
      4. 法的な判断や最終的な査定結果は提示しない
      5. 回答の根拠となる出典を必ず明記する
    PROMPT
  end
end

# app/services/claims_agent_orchestrator.rb
class ClaimsAgentOrchestrator
  def initialize(claim)
    @claim = claim
    @context = { claim_id: claim.id, steps: [] }
  end
 
  def process
    doc_type   = ClassifyDocumentAgent.new(@claim.document).call
    extracted  = ExtractInformationAgent.new(
      document: @claim.document, doc_type: doc_type,
      schema: extraction_schema_for(doc_type)
    ).call
 
    query_embedding = EmbeddingService.new.embed(build_search_query(extracted))
    similar = DocumentChunk.search_by_similarity(query_embedding, limit: 10)
 
    assessment = GenerateAssessmentAgent.new(
      claim_data: extracted, similar_cases: similar,
      guidelines: load_guidelines
    ).call
 
    validated = GuardrailService.new.validate(assessment, context: @context)
 
    ClaimAssessmentResult.create!(
      claim: @claim,
      assessment: validated[:assessment],
      confidence: validated[:confidence],
      similar_case_ids: similar.map(&:id),
      requires_human_review: validated[:confidence] < 0.85,
      processing_context: @context
    )
  end
end

# app/services/eval_runner.rb
class EvalRunner
  EVAL_PROMPT = <<~PROMPT
    あなたは AI の回答品質を評価する審査員です。
    0.0〜1.0 のスコアを付けてください。
 
    ## 評価基準
    - faithfulness: 参照文書に忠実か
    - relevance: 質問に適切に答えているか
    - completeness: 必要な情報が網羅されているか
    - safety: 不適切な内容を含んでいないか
 
    質問: %{question}
    参照文書: %{context}
    AI の回答: %{answer}
 
    JSON で回答: {"faithfulness":0.0,"relevance":0.0,"completeness":0.0,"safety":0.0,"reasoning":""}
  PROMPT
 
  def initialize(llm_client: BedrockLlmClient.new)
    @llm_client = llm_client
  end
 
  def evaluate(question:, context:, answer:)
    prompt = format(EVAL_PROMPT, question: question,
                    context: context, answer: answer)
    scores = JSON.parse(@llm_client.generate(prompt))
 
    EvalResult.create!(
      question: question, context: context, answer: answer,
      faithfulness: scores["faithfulness"],
      relevance: scores["relevance"],
      completeness: scores["completeness"],
      safety: scores["safety"],
      reasoning: scores["reasoning"]
    )
  end
 
  def run_suite(test_cases)
    results = test_cases.map do |tc|
      answer = RagQueryService.new.query(tc[:question])
      evaluate(question: tc[:question], context: tc[:context], answer: answer)
    end
    EvalSuiteReport.new(results)
  end
end

# app/clients/bedrock_llm_client.rb
class BedrockLlmClient
  MODEL_ID = "anthropic.claude-3-5-sonnet-20241022-v2:0"
 
  def initialize(region: "ap-northeast-1")
    @client = Aws::BedrockRuntime::Client.new(region: region)
  end
 
  def model_id = MODEL_ID
 
  def generate(system: nil, messages:, max_tokens: 2048)
    body = { anthropic_version: "bedrock-2023-05-31",
             max_tokens: max_tokens, messages: messages }
    body[:system] = system if system
 
    response = @client.invoke_model(
      model_id: MODEL_ID, content_type: "application/json",
      accept: "application/json", body: body.to_json
    )
    JSON.parse(response.body.read).dig("content", 0, "text")
  end
end

# app/jobs/document_indexing_job.rb
class DocumentIndexingJob < ApplicationJob
  queue_as :ai_indexing
  retry_on Aws::BedrockRuntime::Errors::ThrottlingException,
           wait: :polynomially_longer, attempts: 5
 
  def perform(document_id)
    document = Document.find(document_id)
    document.update!(indexing_status: "processing")
 
    chunks = DocumentChunker.new(strategy: :recursive).chunk(document)
    service = EmbeddingService.new(provider: :bedrock)
 
    chunks.each_with_index do |text, i|
      DocumentChunk.create!(
        document: document, content: text, position: i,
        embedding: service.embed(text),
        metadata: { source: document.filename,
                    section: detect_section(text) }
      )
    end
 
    document.update!(indexing_status: "completed",
                     chunk_count: chunks.size, indexed_at: Time.current)
  rescue StandardError => e
    document.update!(indexing_status: "failed", indexing_error: e.message)
    raise
  end
end

# app/services/guardrail_service.rb
class GuardrailService
  CONFIDENCE_THRESHOLD = 0.85
  AMOUNT_REVIEW_THRESHOLD = 500_000  # 50万円以上は人間レビュー必須
  PROHIBITED = [/確実に支払われます/, /絶対に/, /保証します/, /法的責任/].freeze
 
  def validate(assessment, context: {})
    violations = []
    violations.concat(check_prohibited(assessment[:text]))
    violations << { type: "low_confidence" } if assessment[:confidence] < CONFIDENCE_THRESHOLD
    if assessment[:estimated_amount]&.> AMOUNT_REVIEW_THRESHOLD
      violations << { type: "high_amount", amount: assessment[:estimated_amount] }
    end
    violations.concat(check_hallucination(assessment[:text], context))
 
    AuditLog.create!(event_type: "guardrail_check", claim_id: context[:claim_id],
                     details: { confidence: assessment[:confidence],
                                violations: violations })
 
    { assessment: assessment[:text], confidence: assessment[:confidence],
      violations: violations, requires_human_review: violations.any? }
  end
 
  private
 
  def check_prohibited(text)
    PROHIBITED.filter_map do |pat|
      { type: "prohibited_content", pattern: pat.source } if text.match?(pat)
    end
  end
 
  def check_hallucination(text, context)
    numbers = text.scan(/[\d,]+(?:円|万円|%)/)
    ctx = context[:similar_cases]&.map(&:content)&.join(" ") || ""
    numbers.filter_map do |num|
      { type: "potential_hallucination", value: num } unless ctx.include?(num.tr(",", ""))
    end
  end
end

# app/services/ai_monitoring_service.rb
class AiMonitoringService
  def daily_report
    convos = Conversation.where(created_at: 1.day.ago..)
    evals = EvalResult.where(created_at: 1.day.ago..)
 
    {
      total_queries: convos.count,
      avg_latency_ms: convos.average(:latency_ms)&.round,
      avg_faithfulness: evals.average(:faithfulness)&.round(3),
      low_confidence_rate: percentage(convos, "confidence < 0.85"),
      human_escalation_rate: percentage(convos.where(escalated_to_human: true)),
      drift_detected: detect_drift(evals)
    }
  end
 
  private
 
  def percentage(scope, condition = nil)
    total = scope.count
    return 0.0 if total.zero?
    sub = condition ? scope.where(condition) : scope
    (sub.count.to_f / total * 100).round(1)
  end
 
  def detect_drift(evals)
    return false if evals.count < 50
    recent = evals.order(created_at: :desc).limit(50).average(:faithfulness)
    baseline = EvalResult.where(created_at: 30.days.ago..7.days.ago).average(:faithfulness)
    baseline && (baseline - recent) > 0.05
  end
end

# lib/templates/insurance_rag_template.rb
module Templates
  class InsuranceRagTemplate
    CHUNKING  = { strategy: :recursive, chunk_size: 512, overlap: 64,
                  separators: ["条", "項", "号"] }.freeze
    GUARDRAIL = { confidence_threshold: 0.85,
                  prohibited_patterns: [/確実/, /保証/, /絶対/],
                  require_human_review_for: %w[death disability] }.freeze
    EVAL_SUITE = [
      { name: "基本請求", question: "入院給付金の請求に必要な書類は？",
        expected_topics: %w[診断書 請求書 本人確認書類] },
      { name: "免責事由", question: "既往症がある場合、保険金は支払われますか？",
        expected_topics: %w[告知義務 免責期間 条件付き] }
    ].freeze
 
    def self.apply(project)
      project.configure_chunking(CHUNKING)
      project.configure_guardrails(GUARDRAIL)
      project.load_eval_suite(EVAL_SUITE)
    end
  end
end

# app/services/prompt_builder.rb
class PromptBuilder
  def self.build_system_prompt(customer:, few_shot_examples: [])
    base = <<~PROMPT
      あなたは#{customer.name}の保険査定支援AIです。
      - 参照文書の事実のみに基づいて回答する
      - 「おそらく」「可能性があります」等の曖昧表現を使い断定を避ける
      - 最終的な査定判断は行わず参考情報の提示に留める
      - 個人情報を回答に含めない
      - 金額は参照文書を正確に引用し独自の計算をしない
    PROMPT
 
    if few_shot_examples.any?
      base += "\n## 回答例\n"
      few_shot_examples.each { |ex| base += "Q: #{ex[:question]}\nA: #{ex[:answer]}\n\n" }
    end
    base
  end
end