LLM Routing¶

Cost range per patient journey: $0.07 - $0.50
  - Lower end: Text-based PDFs (no Vision OCR), simple conditions
  - Upper end: Scanned documents (Vision OCR), complex multi-condition cases
  - Average: ~$0.15

# config/model_registry.yaml
models:
  claude-haiku-4.5:
    provider: anthropic
    model_id: claude-haiku-4-5-20250514
    max_tokens: 4096
    temperature: 0.3
    tier: economy
    cost_per_1k_input: 0.001
    cost_per_1k_output: 0.005
    rate_limit_rpm: 1000
    timeout_seconds: 30

  claude-sonnet-4.6:
    provider: anthropic
    model_id: claude-sonnet-4-6-20250514
    max_tokens: 8192
    temperature: 0.1
    tier: premium
    cost_per_1k_input: 0.003
    cost_per_1k_output: 0.015
    rate_limit_rpm: 500
    timeout_seconds: 60

  gpt-4o-mini:
    provider: openai
    model_id: gpt-4o-mini
    max_tokens: 4096
    temperature: 0.2
    tier: bulk
    cost_per_1k_input: 0.00015
    cost_per_1k_output: 0.0006
    rate_limit_rpm: 2000
    timeout_seconds: 15

# Task-to-model routing
routing:
  clinical_extraction:
    primary: claude-sonnet-4.6
    fallback: claude-haiku-4.5
    ab_split:
      claude-sonnet-4.6: 100
      claude-haiku-4.5: 0

  patient_conversation:
    primary: claude-haiku-4.5
    fallback: gpt-4o-mini
    ab_split:
      claude-haiku-4.5: 100

  intent_classification:
    primary: gpt-4o-mini
    fallback: claude-haiku-4.5
    ab_split:
      gpt-4o-mini: 80
      claude-haiku-4.5: 20

  match_explanation:
    primary: claude-haiku-4.5
    fallback: null  # Template fallback, no LLM needed
    ab_split:
      claude-haiku-4.5: 100

  document_reranking:
    primary: claude-haiku-4.5
    fallback: null  # Cosine similarity only
    ab_split:
      claude-haiku-4.5: 100

  vision_ocr:
    primary: claude-sonnet-4.6
    fallback: null  # Only used when other OCR methods fail
    ab_split:
      claude-sonnet-4.6: 100

intent_classification:
  primary: gpt-4o-mini
  ab_split:
    gpt-4o-mini: 80    # 80% of requests use GPT-4o mini
    claude-haiku-4.5: 20  # 20% use Haiku for comparison

async def route_llm_call(
    task: str,
    messages: list[dict],
    tenant_id: str,
) -> LLMResponse:
    """Route an LLM call based on task type and registry config."""
    config = model_registry.get_routing(task)
    model = select_model_ab(config)

    try:
        response = await call_model(model, messages, config)
        await track_usage(task, model, response, tenant_id)
        return response
    except (TimeoutError, RateLimitError, APIError) as e:
        logger.warning(f"Primary model {model} failed for {task}: {e}")

        if config.fallback:
            fallback_response = await call_model(config.fallback, messages, config)
            await track_usage(task, config.fallback, fallback_response, tenant_id, fallback=True)
            return fallback_response

        raise LLMRoutingError(f"All models failed for task {task}")

from langfuse import Langfuse

langfuse = Langfuse()

async def get_prompt(name: str, version: Optional[int] = None) -> str:
    """Fetch a versioned prompt from Langfuse."""
    prompt = langfuse.get_prompt(
        name=name,
        version=version,        # None = latest version
        cache_ttl_seconds=300,  # Cache for 5 minutes
    )
    return prompt.compile()

# Example: clinical_entity_extraction prompt (v3)
"""
You are a medical document analysis specialist. Extract clinical entities
from the following medical document text.

## Rules
1. Extract ALL conditions, procedures, medications, lab results, and vitals
2. Include laterality (left/right/bilateral) when mentioned
3. Include dates when available
4. Include severity/stage when mentioned
5. Do NOT infer information that is not explicitly stated
6. Return results in the specified JSON schema

## Output Schema
{output_schema}

## Document Text
{document_text}
"""

COMORBIDITY_PAIRS = {
    ("E11", "I10"): {"name": "Diabetes + Hypertension", "severity": "moderate"},
    ("E66", "G47.3"): {"name": "Obesity + Sleep Apnea", "severity": "moderate"},
    ("E11", "N18"): {"name": "Diabetes + CKD", "severity": "high"},
    ("I25", "E78"): {"name": "CAD + Hyperlipidemia", "severity": "moderate"},
    # ... 146 more pairs
}

def detect_comorbidities(condition_codes: list[str]) -> list[dict]:
    """Detect comorbidity pairs from extracted condition codes."""
    detected = []
    for i, code1 in enumerate(condition_codes):
        for code2 in condition_codes[i+1:]:
            key = tuple(sorted([code1[:3], code2[:3]]))  # Match on 3-char prefix
            if key in COMORBIDITY_PAIRS:
                detected.append(COMORBIDITY_PAIRS[key])
    return detected

@langfuse.observe(name="clinical_extraction")
async def extract_clinical_entities(text: str, tenant_id: str):
    """Extract clinical entities with full Langfuse tracing."""
    prompt = await get_prompt("clinical_entity_extraction")
    model_config = model_registry.get_routing("clinical_extraction")

    response = await call_model(
        model=model_config.primary,
        messages=[
            {"role": "system", "content": prompt},
            {"role": "user", "content": text},
        ],
    )

    # Langfuse automatically tracks: model, tokens, cost, latency, I/O
    return parse_extraction_response(response)

+---------------------------+--------+--------+---------+
| Category                  | Calls  | Avg $  | Total $ |
+---------------------------+--------+--------+---------+
| Clinical Extraction       |    100 | $0.025 |   $2.50 |
| Patient Conversations     |    300 | $0.003 |   $0.90 |
| Match Explanations        |    250 | $0.001 |   $0.25 |
| Document Reranking        |     50 | $0.005 |   $0.25 |
| Vision OCR (fallback)     |     10 | $0.040 |   $0.40 |
| Intent Classification     |    300 | $0.001 |   $0.30 |
+---------------------------+--------+--------+---------+
| TOTAL                     |  1,010 |        |   $4.60 |
+---------------------------+--------+--------+---------+