HTTP client timeout = 2s
gateway timeout = 1s
retry max attempts = 3
deadline remaining = 800ms
DB statement timeout = 10s
bulkhead wait = 500ms

circuit breaker opens after 50% failures
retry attempts 3x
rate limiter only limits original calls
fallback returns stale data without freshness label

timeout: 500ms

This operation may spend at most 500 ms end-to-end.
It may retry once only for transient failures.
It must use idempotency key if command retry is enabled.
It fails fast when bulkhead is full.
It may return stale cache for reads up to 5 minutes.
It must not fallback to fake success for commands.

platform default
< service default
< dependency default
< operation policy
< safe incident override

public record CommunicationPolicy(
    String dependency,
    Map<String, OperationPolicy> operations,
    DependencyDefaults defaults
) {}

public record OperationPolicy(
    String operationName,
    OperationSemantics semantics,
    DeadlinePolicy deadline,
    TimeoutPolicy timeout,
    RetryPolicy retry,
    CircuitBreakerPolicy circuitBreaker,
    BulkheadPolicy bulkhead,
    RateLimitPolicy rateLimit,
    HedgingPolicy hedging,
    FallbackPolicy fallback,
    ObservabilityPolicy observability
) {}

public record OperationSemantics(
    boolean readOnly,
    boolean sideEffecting,
    boolean idempotentByNature,
    boolean idempotencyKeyRequired,
    boolean strongConsistencyRequired,
    Priority defaultPriority
) {}

retry enabled on side-effecting command without idempotency key
hedging enabled on side-effecting command
fallback default success enabled on command
timeout longer than deadline
bulkhead wait longer than operation budget
retry max attempts impossible within deadline
stale fallback enabled without max staleness
metrics disabled for critical dependency
circuit breaker records validation errors as dependency failure

public final class CommunicationPolicyValidator {
    public void validate(OperationPolicy policy) {
        if (policy.semantics().sideEffecting()
            && policy.retry().enabled()
            && !policy.semantics().idempotencyKeyRequired()) {
            throw new InvalidPolicyException(
                policy.operationName() + " retries side-effecting command without idempotency"
            );
        }

        if (policy.semantics().sideEffecting() && policy.hedging().enabled()) {
            throw new InvalidPolicyException(
                policy.operationName() + " enables hedging for side-effecting command"
            );
        }

        if (policy.fallback().staleCacheAllowed()
            && policy.fallback().maxStaleness() == null) {
            throw new InvalidPolicyException(
                policy.operationName() + " stale fallback requires maxStaleness"
            );
        }

        if (policy.timeout().responseTimeout()
            .compareTo(policy.deadline().defaultDeadline()) > 0) {
            throw new InvalidPolicyException(
                policy.operationName() + " response timeout exceeds default deadline"
            );
        }
    }
}

communication:
  serviceName: workflow-service
  environment: production

  defaults:
    deadline:
      inboundHeader: X-Request-Deadline
      defaultMs: 500
      maxMs: 1000
      minUsefulMs: 75
      reserveResponseMarginMs: 25

    observability:
      metricsEnabled: true
      tracesEnabled: true
      logsEnabled: true
      recordAttemptMetrics: true
      recordPolicyDecisions: true

  dependencies:
    case-service:
      baseUrl: https://case-service.internal
      apiVersion: v1
      owner: case-platform
      criticality: high

      defaults:
        connectionPool:
          maxConnections: 100
          maxConnectionsPerRoute: 50
          acquisitionTimeoutMs: 25
        timeout:
          connectMs: 75
          responseMs: 400
        circuitBreaker:
          slidingWindowType: COUNT_BASED
          slidingWindowSize: 100
          minimumNumberOfCalls: 50
          failureRateThreshold: 50
          slowCallRateThreshold: 50
          slowCallDurationMs: 500
          waitDurationOpenMs: 20000
          permittedHalfOpenCalls: 5
        bulkhead:
          type: semaphore
          maxConcurrentCalls: 50
          maxWaitMs: 0

      operations:
        getCase:
          method: GET
          route: /v1/cases/{caseId}
          semantics:
            readOnly: true
            sideEffecting: false
            idempotentByNature: true
            strongConsistencyRequired: true
            defaultPriority: user-facing

          deadline:
            defaultMs: 300
            maxMs: 600
            minUsefulMs: 50

          timeout:
            connectMs: 50
            responseMs: 250

          retry:
            enabled: true
            maxAttempts: 2
            baseDelayMs: 30
            maxDelayMs: 120
            jitter: full
            retryableStatuses: [429, 502, 503, 504]
            retryBudgetRatio: 0.10
            deadlineAware: true

          circuitBreaker:
            enabled: true
            recordStatuses: [502, 503, 504]
            ignoreStatuses: [400, 401, 403, 404, 409, 422]

          bulkhead:
            enabled: true
            maxConcurrentCalls: 80
            maxWaitMs: 10

          rateLimit:
            enabled: true
            limitForPeriod: 300
            limitRefreshPeriodMs: 1000
            timeoutMs: 0

          hedging:
            enabled: false
            reason: strong-consistency-required

          fallback:
            type: fail-fast
            staleCacheAllowed: false

          observability:
            operationMetric: case.get_case
            traceSpanName: GET /v1/cases/{caseId}

        createEscalation:
          method: POST
          route: /v1/case-escalations
          semantics:
            readOnly: false
            sideEffecting: true
            idempotentByNature: false
            idempotencyKeyRequired: true
            strongConsistencyRequired: true
            defaultPriority: critical-command

          deadline:
            defaultMs: 600
            maxMs: 1000
            minUsefulMs: 100

          timeout:
            connectMs: 75
            responseMs: 450

          retry:
            enabled: true
            maxAttempts: 2
            requiresIdempotencyKey: true
            sameIdempotencyKeyAcrossAttempts: true
            retryableStatuses: [429, 502, 503]
            nonRetryableStatuses: [400, 401, 403, 404, 409, 422]
            unknownOutcomeHandling: dedup-replay-required
            deadlineAware: true

          circuitBreaker:
            enabled: true
            failureRateThreshold: 40
            slowCallDurationMs: 600
            ignoreStatuses: [400, 401, 403, 404, 409, 422]

          bulkhead:
            enabled: true
            maxConcurrentCalls: 40
            maxWaitMs: 0

          rateLimit:
            enabled: true
            limitForPeriod: 100
            limitRefreshPeriodMs: 1000
            timeoutMs: 0

          hedging:
            enabled: false
            reason: side-effecting-command

          fallback:
            type: fail-fast
            allowAsyncHandoff: false
            fakeSuccessAllowed: false

          observability:
            operationMetric: case.create_escalation
            traceSpanName: POST /v1/case-escalations
            recordIdempotencyPresence: true

request context / deadline
→ rate limiter
→ bulkhead
→ circuit breaker
→ retry executor
→ per-attempt timeout
→ transport
→ error mapper
→ fallback

public record RemoteCallAttempt(
    int attemptNumber,
    boolean hedgeAttempt,
    Instant startedAt,
    Duration timeout,
    Optional<Integer> statusCode,
    Optional<String> errorCode,
    AttemptOutcome outcome
) {}

public record RemoteCallResult<T>(
    T value,
    boolean degraded,
    int attempts,
    Duration totalDuration,
    Optional<String> fallbackType
) {}

logical call success
attempt success
success after retry
success via fallback
failure due to circuit open
failure due to bulkhead full
failure due to deadline exceeded

public enum FailureClass {
    CALLER_BAD_REQUEST,
    AUTHENTICATION,
    AUTHORIZATION,
    NOT_FOUND,
    DOMAIN_CONFLICT,
    PRECONDITION_FAILED,
    DOMAIN_VALIDATION,
    RATE_LIMITED,
    REMOTE_UNAVAILABLE,
    REMOTE_TIMEOUT,
    CONNECT_TIMEOUT,
    READ_TIMEOUT,
    DEADLINE_EXCEEDED,
    BULKHEAD_FULL,
    CIRCUIT_OPEN,
    LOCAL_RATE_LIMITED,
    UNKNOWN
}

public final class RemoteOperationExecutor {
    private final PolicyRegistry policyRegistry;
    private final Telemetry telemetry;

    public <T> T execute(
        String dependency,
        String operation,
        RequestContext context,
        Supplier<T> transportCall
    ) {
        OperationPolicy policy = policyRegistry.get(dependency, operation);
        policy.validate();

        if (!context.deadline().canFit(policy.deadline().minUseful())) {
            throw new DeadlineTooShortException();
        }

        return telemetry.observeLogicalCall(policy, context, () ->
            executeWithPolicy(policy, context, transportCall)
        );
    }

    private <T> T executeWithPolicy(
        OperationPolicy policy,
        RequestContext context,
        Supplier<T> transportCall
    ) {
        Supplier<T> supplier = () -> executeAttempts(policy, context, transportCall);

        supplier = applyCircuitBreaker(policy, supplier);
        supplier = applyBulkhead(policy, supplier);
        supplier = applyRateLimiter(policy, supplier);

        try {
            return supplier.get();
        } catch (Throwable failure) {
            return applyFallback(policy, context, failure);
        }
    }
}

private <T> T executeAttempts(
    OperationPolicy policy,
    RequestContext context,
    Supplier<T> call
) {
    Throwable lastFailure = null;

    for (int attempt = 1; attempt <= policy.retry().maxAttempts(); attempt++) {
        Duration attemptTimeout = context.deadline().timeoutWithMargin(
            policy.timeout().responseTimeout(),
            policy.deadline().reserveResponseMargin()
        );

        if (attemptTimeout.isZero()) {
            throw new DeadlineExceededException();
        }

        try {
            return executeOneAttemptWithTimeout(call, attemptTimeout);
        } catch (Throwable failure) {
            lastFailure = failure;

            RetryDecision decision = retryDecider.decide(
                policy,
                context,
                failure,
                attempt
            );

            if (!decision.shouldRetry()) {
                throw failure;
            }

            sleepWithoutHoldingBulkheadPermit(decision.delay());
        }
    }

    throw new RetryExhaustedException(lastFailure);
}

public interface PolicyRegistry {
    OperationPolicy get(String dependency, String operation);
}

public final class ValidatingPolicyRegistry implements PolicyRegistry {
    private final Map<OperationKey, OperationPolicy> policies;
    private final CommunicationPolicyValidator validator;

    public ValidatingPolicyRegistry(
        Map<OperationKey, OperationPolicy> policies,
        CommunicationPolicyValidator validator
    ) {
        this.policies = Map.copyOf(policies);
        this.validator = validator;

        this.policies.values().forEach(validator::validate);
    }

    @Override
    public OperationPolicy get(String dependency, String operation) {
        OperationPolicy policy = policies.get(new OperationKey(dependency, operation));
        if (policy == null) {
            throw new MissingCommunicationPolicyException(dependency, operation);
        }
        return policy;
    }
}

platformDefaults:
  allOutboundCalls:
    timeoutRequired: true
    deadlinePropagationRequired: true
    observabilityRequired: true
    connectionPoolRequired: true
    errorMappingRequired: true

  unsafeCommands:
    retryDisabledUnlessIdempotencyKeyRequired: true
    hedgingForbidden: true
    fakeSuccessFallbackForbidden: true

  allOperations:
    maxDeadlineMs: 2000
    maxRetryAttempts: 2
    maxBulkheadWaitMs: 50
    metricsCardinalityGuard: true

runtimeOverrides:
  hedging.enabled: false
  retry.enabled: false
  fallback.staleCache.enabled: false
  loadShedding.forceLevel: DEGRADED
  circuitBreaker.forceOpen:
    - external-provider.submitDocument
  circuitBreaker.disable:
    - noncritical-recommendation.getSuggestions

remote.logical.calls.total{dependency,operation,outcome}
remote.logical.duration{dependency,operation,outcome}
remote.attempts.total{dependency,operation,attempt_kind,outcome}
remote.timeouts.total{dependency,operation,type}
remote.retries.total{dependency,operation,decision}
remote.circuit.state{dependency,operation}
remote.bulkhead.rejections.total{dependency,operation}
remote.rate_limit.denied.total{dependency,operation}
remote.fallback.used.total{dependency,operation,type}
remote.deadline.remaining_ms{dependency,operation}

Is the dependency healthy?
Is the caller protected?
Are users seeing fresh success, degraded success, or failure?
Is resilience policy helping or hurting?

# Runbook: case-service.createEscalation communication failure

## Symptoms
- timeout rate above 5%
- circuit breaker open
- bulkhead rejection > 10%
- fallback not allowed
- upstream workflows failing

## First checks
1. Check dependency health dashboard.
2. Check recent deploys for caller and provider.
3. Check policy version and recent overrides.
4. Check timeout vs deadline remaining.
5. Check retry volume and retry budget.
6. Check idempotency-key presence.
7. Check outbox/reconciliation backlog.

## Safe mitigations
- reduce retry attempts if retry storm
- force circuit open if dependency is collapsing
- shed batch traffic
- increase bulkhead only if provider has capacity
- route to async durable intent only if approved
- do not enable fake success fallback

## Unsafe mitigations
- disabling idempotency requirement
- retrying commands with new keys
- increasing timeout above gateway deadline
- returning success without provider confirmation

operations:
  createEscalation:
    openapiOperationId: createCaseEscalation
    method: POST
    route: /v1/case-escalations

app retry maxAttempts=2
mesh retry maxAttempts=3
gateway retry maxAttempts=2

2 × 3 × 2 = 12 attempts

environments:
  production:
    case-service.getCase.bulkhead.maxConcurrentCalls: 80
  staging:
    case-service.getCase.bulkhead.maxConcurrentCalls: 10

@Test
void rejectsRetryForSideEffectingCommandWithoutIdempotency() {
    OperationPolicy policy = policyBuilder()
        .sideEffecting(true)
        .idempotencyKeyRequired(false)
        .retryEnabled(true)
        .build();

    assertThatThrownBy(() -> validator.validate(policy))
        .isInstanceOf(InvalidPolicyException.class);
}

@Test
void createEscalationRetriesWithSameIdempotencyKey() {
    stub.transientFailureThenSuccess();

    client.createEscalation(command);

    stub.verifyPostCount(2);
    stub.verifySameHeaderAcrossAttempts("Idempotency-Key");
}

communication:
  policyVersion: 2026-07-05.1

communication.policy.version=2026-07-05.1

Did behavior change because code changed or policy changed?

# Dependency: case-service

Owner: case-platform
Criticality: high

## Operation: getCase
- Deadline: 300 ms default, 600 ms max
- Retry: enabled, max 2 attempts
- Circuit breaker: enabled
- Bulkhead: 80 concurrent
- Fallback: fail-fast
- Hedging: disabled due to strong consistency
- Error retryability: 429/502/503/504
- Dashboard: ...
- Runbook: ...

operation:
  identity:
    dependency:
    operation:
    method:
    route:
    owner:
    criticality:

  semantics:
    readOnly:
    sideEffecting:
    idempotencyKeyRequired:
    consistency:
    priority:

  deadline:
    defaultMs:
    maxMs:
    minUsefulMs:
    propagate:

  timeout:
    connectMs:
    poolAcquireMs:
    responseMs:
    totalAttemptMs:

  retry:
    enabled:
    maxAttempts:
    retryableFailures:
    nonRetryableFailures:
    backoff:
    jitter:
    budget:
    idempotencyRequired:

  circuitBreaker:
    enabled:
    window:
    minimumCalls:
    failureRate:
    slowCallRate:
    halfOpen:

  bulkhead:
    enabled:
    type:
    maxConcurrent:
    maxWait:
    queueCapacity:

  rateLimit:
    enabled:
    dimensions:
    rate:
    burst:
    wait:

  hedging:
    enabled:
    delay:
    budget:
    suppressionRules:

  fallback:
    type:
    maxStaleness:
    partialAllowed:
    fakeSuccessAllowed:
    asyncHandoff:

  observability:
    metrics:
    traces:
    logs:
    dashboard:
    alerts:
    runbook:

deadline decides whether work is still useful
timeout bounds each wait
retry handles transient failure
circuit breaker stops repeated failure
bulkhead isolates resource use
rate limit controls demand
load shedding preserves survival
hedging fights tail latency carefully
fallback preserves semantic usefulness
observability proves what happened

Which library annotation should I add?

What is the communication policy for this operation under success, slowness, overload, partial failure, and unknown outcome?