observability & production monitoring

// GOOD: structured, traceable, searchable
log.info("[traceId={}] | event=orderCreated | orderId={} | userId={} | amount={}",
   RequestContext.getRequestId(), order.getId(), userId, amount);
 
// GOOD: separate data fields by |
log.info("[traceId={}] | event=paymentProcessed | transId={} | status={} | duration={}ms",
   traceId, transactionId, status, duration);
 
// BAD: unstructured, hard to parse
log.info("Order " + order.getId() + " created by user " + userId);
 
// BAD: logging entire objects (PII risk, noise)
log.info("Processing request: {}", requestDTO); // ← dumps all fields

// Set once in filter → available in all logs within request
MDC.put("traceId", requestId);
MDC.put("userId", jwt.getSubject());
MDC.put("service", "order-service");
 
// Log4j2 pattern includes MDC automatically
// %X{traceId} %X{userId} trong log pattern

<Configuration>
   <Properties>
       <Property name="LOG_PATTERN">
           %d{yyyy-MM-dd HH:mm:ss.SSS} [%pid] [%t] %-5level %logger{36}.%M(%L) - %msg%n
       </Property>
   </Properties>
 
   <Appenders>
       <!-- Async appender for performance -->
       <RollingFile name="File" fileName="logs/application.log"
                    filePattern="logs/application-%d{yyyy-MM-dd}-%i.log">
           <PatternLayout pattern="${LOG_PATTERN}"/>
           <Policies>
               <TimeBasedTriggeringPolicy interval="1"/>
               <SizeBasedTriggeringPolicy size="100MB"/>
           </Policies>
           <DefaultRolloverStrategy max="30"/>
       </RollingFile>
   </Appenders>
 
   <Loggers>
       <AsyncRoot level="info">
           <AppenderRef ref="File"/>
       </AsyncRoot>
 
       <!-- Reduce noise from frameworks -->
       <Logger name="org.hibernate.SQL" level="debug"/> <!-- show SQL in dev -->
       <Logger name="org.springframework.web" level="warn"/>
       <Logger name="com.netflix.eureka" level="warn"/>
   </Loggers>
</Configuration>

management:
  endpoints:
    web:
      exposure:
        include: health, info, prometheus, metrics
  endpoint:
    health:
      show-details: when-authorized
  metrics:
    tags:
      application: ${spring.application.name}
      environment: ${spring.profiles.active:dev}
    export:
      prometheus:
        enabled: true

@Service
@RequiredArgsConstructor
public class OrderService {
   private final MeterRegistry meterRegistry;
   private final Counter orderCreatedCounter;
   private final Timer orderProcessingTimer;
 
   public OrderService(MeterRegistry registry) {
       this.meterRegistry = registry;
       this.orderCreatedCounter = Counter.builder("orders.created")
           .description("Total orders created")
           .tag("service", "order-service")
           .register(registry);
       this.orderProcessingTimer = Timer.builder("orders.processing.duration")
           .description("Order processing time")
           .publishPercentiles(0.5, 0.95, 0.99)
           .register(registry);
   }
 
   public Order createOrder(CreateOrderDTO dto) {
       return orderProcessingTimer.record(() -> {
           Order order = processOrder(dto);
           orderCreatedCounter.increment();
           meterRegistry.counter("orders.created.by_type",
               "type", order.getType().name()).increment();
           return order;
       });
   }
}

// Counter — only goes up
Counter.builder("emails.sent")
   .tag("type", "confirmation")
   .register(registry)
   .increment();
 
// Gauge — current state
Gauge.builder("queue.size", queue, Queue::size)
   .register(registry);
 
// Timer — measure duration
Timer.builder("db.query.duration")
   .tag("query", "findByStatus")
   .publishPercentiles(0.5, 0.95, 0.99)
   .register(registry);
 
// Distribution Summary — value distribution
DistributionSummary.builder("http.request.size")
   .baseUnit("bytes")
   .publishPercentiles(0.5, 0.95)
   .register(registry)
   .record(requestBody.length);

@Component
public class ExternalServiceHealthIndicator implements HealthIndicator {
 
   @Override
   public Health health() {
       try {
           boolean reachable = checkExternalService();
           if (reachable) {
               return Health.up()
                   .withDetail("service", "payment-gateway")
                   .withDetail("responseTime", "45ms")
                   .build();
           }
           return Health.down()
               .withDetail("service", "payment-gateway")
               .withDetail("error", "Connection refused")
               .build();
       } catch (Exception e) {
           return Health.down(e).build();
       }
   }
}

management:
  tracing:
    sampling:
      probability: 0.1 # 10% in prod, 1.0 in dev
    propagation:
      type: w3c # W3C TraceContext headers
 
# Export to collector
otel:
  exporter:
    otlp:
      endpoint: http://otel-collector:4318
  resource:
    attributes:
      service.name: ${spring.application.name}
      deployment.environment: ${spring.profiles.active:dev}

@Service
@RequiredArgsConstructor
public class PaymentService {
   private final ObservationRegistry observationRegistry;
 
   public PaymentResult processPayment(PaymentRequest request) {
       return Observation.createNotStarted("payment.process", observationRegistry)
           .lowCardinalityKeyValue("payment.type", request.getType().name())
           .highCardinalityKeyValue("payment.id", request.getId().toString())
           .observe(() -> {
               // Auto-creates span with timing
               validate(request);
               PaymentResult result = callGateway(request);
               persistResult(result);
               return result;
           });
   }
}
 
// Or with Tracer directly
@Service
@RequiredArgsConstructor
public class ImportService {
   private final Tracer tracer;
 
   public void importBatch(List<Record> records) {
       Span span = tracer.nextSpan().name("import.batch").start();
       try (Tracer.SpanInScope ws = tracer.withSpan(span)) {
           span.tag("batch.size", String.valueOf(records.size()));
           records.forEach(this::importRecord);
           span.tag("batch.result", "success");
       } catch (Exception e) {
           span.error(e);
           throw e;
       } finally {
           span.end();
       }
   }
}

// HTTP client — auto-propagated via WebClient/RestClient instrumentation
// RabbitMQ — propagated via message headers
// Manual propagation (e.g., custom protocol):
String traceParent = tracer.currentSpan().context().traceId();
// Include in outgoing request header

# Error rate > 5% for 5 minutes
- alert: HighErrorRate
 expr: |
   sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
   / sum(rate(http_server_requests_seconds_count[5m])) > 0.05
 for: 5m
 labels:
   severity: P2
 
# P95 latency > 2 seconds
- alert: HighLatency
 expr: |
   histogram_quantile(0.95, rate(http_server_requests_seconds_bucket[5m])) > 2
 for: 5m
 labels:
   severity: P2
 
# DB connection pool exhausted
- alert: DbPoolExhausted
 expr: hikaricp_connections_pending > 0
 for: 2m
 labels:
   severity: P1
 
# Service health check down
- alert: ServiceDown
 expr: up{job="order-service"} == 0
 for: 1m
 labels:
   severity: P1
 
# JVM heap > 85%
- alert: HighMemoryUsage
 expr: |
   jvm_memory_used_bytes{area="heap"}
   / jvm_memory_max_bytes{area="heap"} > 0.85
 for: 10m
 labels:
   severity: P3
 
# Queue growing (messages not consumed)
- alert: QueueBacklog
 expr: rabbitmq_queue_messages > 10000
 for: 10m
 labels:
   severity: P3

# Change log level without restart via Actuator
curl -X POST http://service:8080/actuator/loggers/com.vpbank.internal \
 -H 'Content-Type: application/json' \
 -d '{"configuredLevel": "DEBUG"}'
 
# Revert
curl -X POST http://service:8080/actuator/loggers/com.vpbank.internal \
 -H 'Content-Type: application/json' \
 -d '{"configuredLevel": "INFO"}'

# Via Actuator
curl http://service:8080/actuator/threaddump
 
# Look for:
# - BLOCKED threads (deadlock)
# - WAITING on same lock (contention)
# - Many threads in same stack (bottleneck)

# Via Actuator
curl -o heapdump.hprof http://service:8080/actuator/heapdump
 
# Analyze with Eclipse MAT or VisualVM
# Look for:
# - Largest retained objects
# - Objects growing over time
# - Unclosed resources (connections, streams)