Browser
  -> Gateway HTTP Server Span
  -> System HTTP Server Span
  -> Feign Client Span
  -> Infra HTTP Server Span
  -> JDBC Client Span

log.info("[getPermissionInfo][OpenTelemetry 链路验证，userId({})]", userId);

<properties>
    <opentelemetry.version>1.64.0</opentelemetry.version>
</properties>

<dependencyManagement>
    <dependencies>
        <dependency>
            <groupId>io.opentelemetry</groupId>
            <artifactId>opentelemetry-api</artifactId>
            <version>${opentelemetry.version}</version>
        </dependency>
    </dependencies>
</dependencyManagement>

mkdir -p .local/otel

curl -fL \
  https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.28.1/opentelemetry-javaagent.jar \
  -o .local/otel/opentelemetry-javaagent.jar

echo "faa89bdeebf9b1f52be4a4374689176717b02a59df2d8f8b6eb9aa39f9292589  .local/otel/opentelemetry-javaagent.jar" \
  | sha256sum -c -

.local/otel/opentelemetry-javaagent.jar: OK

-javaagent:/absolute/path/opentelemetry-javaagent.jar
-Dotel.service.name=system-server
-Dotel.exporter.otlp.endpoint=http://127.0.0.1:4318
-Dotel.exporter.otlp.protocol=http/protobuf
-Dotel.propagators=tracecontext,baggage
-Dotel.traces.exporter=otlp
-Dotel.logs.exporter=otlp
-Dotel.metrics.exporter=none

-Dotel.service.name=infra-server

-Dotel.service.name=gateway-server

[otel.javaagent] opentelemetry-javaagent - version: 2.28.1

FROM alpine:3.22 AS otel-agent
ARG OTEL_JAVA_AGENT_VERSION=2.28.1
ARG OTEL_JAVA_AGENT_SHA256=faa89bdeebf9b1f52be4a4374689176717b02a59df2d8f8b6eb9aa39f9292589

RUN wget -q -O /opentelemetry-javaagent.jar \
    "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_JAVA_AGENT_VERSION}/opentelemetry-javaagent.jar" \
    && echo "${OTEL_JAVA_AGENT_SHA256}  /opentelemetry-javaagent.jar" | sha256sum -c -

FROM eclipse-temurin:25-jre
WORKDIR /app
COPY ./target/application.jar app.jar
COPY --from=otel-agent /opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar

ENV JAVA_TOOL_OPTIONS="-javaagent:/opt/opentelemetry-javaagent.jar" \
    OTEL_SERVICE_NAME="system-server" \
    OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4318" \
    OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" \
    OTEL_PROPAGATORS="tracecontext,baggage" \
    OTEL_TRACES_EXPORTER="otlp" \
    OTEL_LOGS_EXPORTER="otlp" \
    OTEL_METRICS_EXPORTER="none"

CMD java ${JAVA_OPTS} -jar app.jar

public final class TracerUtils {

    public static final String HEADER_TRACE_ID = "trace-id";

    private TracerUtils() {
    }

    public static String getTraceId() {
        SpanContext context = Span.current().getSpanContext();
        return context.isValid() ? context.getTraceId() : "";
    }
}

public class TraceFilter extends OncePerRequestFilter {

    @Override
    protected void doFilterInternal(HttpServletRequest request,
                                    HttpServletResponse response,
                                    FilterChain chain)
            throws IOException, ServletException {
        String traceId = TracerUtils.getTraceId();
        if (!traceId.isEmpty()) {
            response.setHeader(TracerUtils.HEADER_TRACE_ID, traceId);
        }
        chain.doFilter(request, response);
    }
}

@Bean
public FilterRegistrationBean<TraceFilter> traceFilter() {
    FilterRegistrationBean<TraceFilter> bean = new FilterRegistrationBean<>();
    bean.setFilter(new TraceFilter());
    bean.setOrder(WebFilterOrderEnum.TRACE_FILTER);
    return bean;
}

HTTP/1.1 200
trace-id: 8a7c479d422fa7bc0fe1aade0d703a24
Content-Type: application/json;charset=UTF-8

CorsConfiguration config = new CorsConfiguration();
config.setAllowCredentials(true);
config.addAllowedOriginPattern("*");
config.addAllowedHeader("*");
config.addAllowedMethod("*");
config.addExposedHeader(TracerUtils.HEADER_TRACE_ID);

Access-Control-Expose-Headers: trace-id

// Axios
const traceId = response.headers['trace-id']

// Fetch
const traceId = response.headers.get('trace-id')

<property name="CONSOLE_LOG_PATTERN"
          value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] [trace_id=%X{trace_id:-} span_id=%X{span_id:-}] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n"/>

<property name="FILE_LOG_PATTERN"
          value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] [trace_id=%X{trace_id:-} span_id=%X{span_id:-}] %-5level %logger{50}:%L - %msg%n"/>

log.info("[getPermissionInfo][OpenTelemetry 链路验证，userId({})]", userId);

2026-08-23 12:47:10.123 [http-nio-48081-exec-4]
[trace_id=8a7c479d422fa7bc0fe1aade0d703a24 span_id=12b91feec45048af]
INFO  c.h.h.m.s.c.a.a.AuthController:104 -
[getPermissionInfo][OpenTelemetry 链路验证，userId(1)]

@Around("@annotation(trace)")
public Object around(ProceedingJoinPoint joinPoint, BizTrace trace) throws Throwable {
    Span span = tracer.spanBuilder(buildOperationName(joinPoint, trace))
            .setAttribute("component", "biz")
            .startSpan();
    try (Scope ignored = span.makeCurrent()) {
        return joinPoint.proceed();
    } catch (Throwable throwable) {
        span.recordException(throwable);
        span.setStatus(StatusCode.ERROR, throwable.getMessage());
        throw throwable;
    } finally {
        span.end();
    }
}

extensions:
  health_check:
    endpoint: 0.0.0.0:13133

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 384
    spike_limit_mib: 64
  batch:
    timeout: 1s
    send_batch_size: 8192
    send_batch_max_size: 10000

exporters:
  otlphttp/elasticsearch:
    endpoint: http://elasticsearch:9200/_otlp
    compression: gzip
    timeout: 30s
    sending_queue:
      enabled: true
      sizer: bytes
      queue_size: 50000000
      block_on_overflow: true
      batch:
        flush_timeout: 1s
        min_size: 1000000
        max_size: 4000000
    retry_on_failure:
      enabled: true
      initial_interval: 1s
      max_interval: 30s
      max_elapsed_time: 300s

service:
  extensions: [health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlphttp/elasticsearch]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlphttp/elasticsearch]

http://elasticsearch:9200/_otlp

/_otlp/v1/traces
/_otlp/v1/logs

otelcol-contrib validate --config script/docker/otel-collector-config.yaml

logs-generic.otel-default
traces-generic.otel-default

curl 'http://127.0.0.1:9200/_data_stream?pretty'

curl -H 'Content-Type: application/json' \
  'http://127.0.0.1:9200/logs-generic.otel-default/_search?pretty' \
  -d '{
    "size": 1,
    "sort": [{"@timestamp": "desc"}],
    "query": {"match_all": {}}
  }'

<dependency>
    <groupId>org.apache.rocketmq</groupId>
    <artifactId>rocketmq-spring-boot-starter</artifactId>
    <optional>true</optional>
</dependency>

@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(
        prefix = "handao.access-log",
        name = "enable",
        havingValue = "true",
        matchIfMissing = true)
@ConditionalOnProperty(
        prefix = "rocketmq",
        name = {"name-server", "producer.group"})
public ApiAccessLogProducer apiAccessLogProducer(
        RocketMQTemplate rocketMQTemplate,
        @Value("${handao.access-log.rocketmq.topic:api-access-log}") String topic) {
    return new ApiAccessLogProducer(rocketMQTemplate, topic);
}

@Slf4j
public class ApiAccessLogProducer {

    private final RocketMQTemplate rocketMQTemplate;
    private final String topic;

    public void send(ApiAccessLogCreateReqDTO dto) {
        String message = null;
        try {
            message = JsonUtils.toJsonString(dto);
            String finalMessage = message;
            rocketMQTemplate.asyncSend(topic, message, new SendCallback() {
                @Override
                public void onSuccess(SendResult result) {
                    // 旁路日志成功时不再打印日志，避免日志放大
                }

                @Override
                public void onException(Throwable throwable) {
                    log.error("[send][访问日志异步发送失败，topic({}) message({})]",
                            topic, StrUtils.maxLength(finalMessage, 500), throwable);
                }
            });
        } catch (Throwable throwable) {
            log.error("[send][访问日志提交发送失败，topic({}) message({})]",
                    topic, StrUtils.maxLength(message, 500), throwable);
        }
    }
}

// 生产端预生成主键，同一消息无论消费多少次，ID 都不变
accessLog.setId(IdUtil.getSnowflakeNextId());
accessLog.setTraceId(TracerUtils.getTraceId());

@NotNull(message = "日志编号不能为空")
private Long id;

@Component
@ConditionalOnProperty(
        prefix = "handao.access-log",
        name = "enable",
        havingValue = "true",
        matchIfMissing = true)
@ConditionalOnProperty(prefix = "rocketmq", name = "name-server")
@RocketMQMessageListener(
        topic = "${handao.access-log.rocketmq.topic:api-access-log}",
        consumerGroup = "${handao.access-log.rocketmq.consumer-group:api-access-log-consumer}")
public class ApiAccessLogMessageConsumer implements RocketMQListener<String> {

    @Resource
    private ApiAccessLogService apiAccessLogService;

    @Override
    public void onMessage(String message) {
        try {
            ApiAccessLogCreateReqDTO dto =
                    JsonUtils.parseObject(message, ApiAccessLogCreateReqDTO.class);
            apiAccessLogService.createApiAccessLog(dto);
        } catch (RuntimeException exception) {
            log.error("[onMessage][API 访问日志落库失败，message({})]", message, exception);
            throw exception;
        }
    }
}

try {
    apiAccessLogMapper.insert(apiAccessLog);
} catch (DuplicateKeyException exception) {
    ApiAccessLogDO existing = TenantUtils.executeIgnore(
            () -> apiAccessLogMapper.selectById(apiAccessLog.getId()));
    if (existing == null) {
        // 可能是其他唯一约束冲突，不能错误地吞掉
        throw exception;
    }
    log.debug("[createApiAccessLog][访问日志({}) 已存在，忽略重复消息]",
            apiAccessLog.getId());
}

public void sendMessageBefore(SendMessageContext context) {
    Long tenantId = TenantContextHolder.getTenantId();
    if (tenantId != null) {
        context.getMessage().putUserProperty("tenant-id", tenantId.toString());
    }
}

public void consumeMessageBefore(ConsumeMessageContext context) {
    String tenantId = context.getMsgList().getFirst().getUserProperty("tenant-id");
    if (StrUtil.isNotEmpty(tenantId)) {
        TenantContextHolder.setTenantId(Long.parseLong(tenantId));
    }
}

public void consumeMessageAfter(ConsumeMessageContext context) {
    TenantContextHolder.clear();
}

rocketmq:
  name-server: 192.168.1.117:9876
  producer:
    group: ${spring.application.name}_PRODUCER

handao:
  access-log:
    enable: true
    rocketmq:
      topic: api-access-log
      consumer-group: api-access-log-consumer

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.2
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
    ports:
      - "9200:9200"
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data

  kibana:
    image: docker.elastic.co/kibana/kibana:9.5.2
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    ports:
      - "5601:5601"
    depends_on:
      elasticsearch:
        condition: service_healthy

  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.159.0
    command: ["--config=/etc/otelcol-contrib/config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
    ports:
      - "4317:4317"
      - "4318:4318"
      - "13133:13133"

  rocketmq-namesrv:
    image: apache/rocketmq:5.5.0
    command: sh mqnamesrv
    ports:
      - "9876:9876"

  rocketmq-broker:
    image: apache/rocketmq:5.5.0
    command: sh mqbroker -c /home/rocketmq/conf/broker.conf --enable-proxy
    environment:
      - NAMESRV_ADDR=rocketmq-namesrv:9876
    ports:
      - "8080:8080"
      - "8081:8081"
      - "10909:10909"
      - "10911:10911"
      - "10912:10912"

cd script/docker
docker compose up -d \
  elasticsearch kibana otel-collector rocketmq-namesrv rocketmq-broker

podman compose up -d \
  elasticsearch kibana otel-collector rocketmq-namesrv rocketmq-broker

brokerIP1=127.0.0.1

brokerIP1=192.168.1.117

MIDDLEWARE_HOST=192.168.1.117

for port in 3306 6379 8848 9876 10911 4318 13133 9200 5601; do
  nc -z -w 2 192.168.1.117 "$port" \
    && echo "OPEN $port" \
    || echo "CLOSED $port"
done

sed 's#http://elasticsearch:9200#http://192.168.1.117:9200#' \
  script/docker/otel-collector-config.yaml \
  > /tmp/handao-otel-collector.yaml

podman run --rm \
  --name handao-otel-collector \
  --network host \
  -v /tmp/handao-otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro,Z \
  otel/opentelemetry-collector-contrib:0.159.0 \
  --config=/etc/otelcol-contrib/config.yaml

curl http://127.0.0.1:13133

GET /admin-api/system/auth/get-permission-info

public CommonResult<AuthPermissionInfoRespVO> getPermissionInfo() {
    Long userId = getLoginUserId();
    log.info("[getPermissionInfo][OpenTelemetry 链路验证，userId({})]", userId);
    // ...
}

curl -i \
  -H 'Authorization: Bearer YOUR_TOKEN' \
  http://127.0.0.1:48081/admin-api/system/auth/get-permission-info

trace-id: 8a7c479d422fa7bc0fe1aade0d703a24

logs-generic.otel-*
traces-generic.otel-*

trace_id : "8a7c479d422fa7bc0fe1aade0d703a24"

body.text : "*OpenTelemetry 链路验证*"

curl -H 'Content-Type: application/json' \
  'http://192.168.1.117:9200/logs-generic.otel-default,traces-generic.otel-default/_search?pretty' \
  -d '{
    "size": 100,
    "sort": [{"@timestamp": "asc"}],
    "query": {
      "term": {
        "trace_id": "8a7c479d422fa7bc0fe1aade0d703a24"
      }
    }
  }'

curl -H 'Content-Type: application/json' \
  'http://192.168.1.117:9200/logs-generic.otel-default/_search?pretty' \
  -d '{
    "size": 20,
    "sort": [{"@timestamp": "desc"}],
    "query": {
      "match_phrase": {
        "body.text": "OpenTelemetry 链路验证"
      }
    }
  }'

{
  "trace_id": "8a7c479d422fa7bc0fe1aade0d703a24",
  "span_id": "12b91feec45048af",
  "severity_text": "INFO",
  "resource": {
    "attributes": {
      "service.name": "system-server"
    }
  },
  "scope": {
    "name": "com.example.system.AuthController"
  },
  "body": {
    "text": "[getPermissionInfo][OpenTelemetry 链路验证，userId(1)]"
  }
}

SELECT id,
       trace_id,
       application_name,
       request_method,
       request_url,
       duration,
       result_code,
       begin_time
FROM infra_api_access_log
ORDER BY id DESC
LIMIT 20;

@Test
void getTraceIdReturnsCurrentOpenTelemetryTraceId() {
    SpanContext context = SpanContext.create(
            "0123456789abcdef0123456789abcdef",
            "0123456789abcdef",
            TraceFlags.getSampled(),
            TraceState.getDefault());

    try (Scope ignored = Span.wrap(context).makeCurrent()) {
        assertEquals(
                "0123456789abcdef0123456789abcdef",
                TracerUtils.getTraceId());
    }
}

@Test
void corsFilterExposesTraceIdHeader() throws Exception {
    MockHttpServletRequest request =
            new MockHttpServletRequest("GET", "/admin-api/test");
    request.addHeader(HttpHeaders.ORIGIN, "http://localhost:3000");
    MockHttpServletResponse response = new MockHttpServletResponse();

    CorsFilter filter = new HandaoWebAutoConfiguration()
            .corsFilterBean().getFilter();

    filter.doFilter(request, response,
            (req, resp) -> ((HttpServletResponse) resp)
                    .setHeader("trace-id", "0123456789abcdef0123456789abcdef"));

    assertEquals("trace-id",
            response.getHeader(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS));
}

apiAccessLogService.createApiAccessLog(createDTO);
apiAccessLogService.createApiAccessLog(createDTO);

assertEquals(1L, apiAccessLogMapper.selectCount());

mvn -pl \
  handao-gateway,\
  handao-module-system/handao-module-system-server,\
  handao-module-infra/handao-module-infra-server,\
  handao-server \
  -am -DskipTests compile

docker compose -f script/docker/docker-compose.yml config

otelcol-contrib validate \
  --config script/docker/otel-collector-config.yaml

ps -ef | grep opentelemetry-javaagent

Access-Control-Expose-Headers: trace-id

trace_id

trace.id

otelcol-contrib validate --config config.yaml

9876
10909
10911
10912

xpack.security.enabled=false

exporters:
  otlphttp/elasticsearch:
    endpoint: https://elasticsearch.example.com:9200/_otlp
    headers:
      Authorization: "ApiKey ${env:ELASTICSEARCH_API_KEY}"
    tls:
      ca_file: /etc/otelcol/certs/ca.crt

extensions:
  health_check:
  file_storage:
    directory: /var/lib/otelcol/storage

exporters:
  otlphttp/elasticsearch:
    sending_queue:
      enabled: true
      storage: file_storage

service:
  extensions: [health_check, file_storage]

OTEL_TRACES_SAMPLER=always_on

OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=0.1