bladex4.5.0的/actuator/prometheus接口返回中缺失了jvm相关的指标

Blade 已结

关注举报

 2  123

北京智鼎优源剑圣 2025-05-15 16:53

一、该问题的重现步骤是什么？

1. bladex cloud版本，从bladex3.4.0升级到bladex4.5.0

2. 除了blade-admin模块之外，其他模块的/actuator/prometheus接口返回中缺失了jvm相关的指标，导致grafana的jvm仪表盘中没有数据

3. 为了排除干扰，从头搭建了bladex官方最新版纯净环境，效果一致，也是除了blade-admin模块之外都缺了jvm的指标

二、你期待的结果是什么？实际看到的又是什么？

我期待的结果是有jvm相关的一系列指标，比如blade-admin模块的返回节选：

 HTTP/1.1 200 OK
Content-Type: text/plain;version=0.0.4;charset=utf-8
Content-Length: 24963
Cache-Control: no-cache, no-store, max-age=0, must-revalidate
Pragma: no-cache
Expires: 0
X-Content-Type-Options: nosniff
X-XSS-Protection: 0
Referrer-Policy: no-referrer
connection: close
 
# HELP jvm_threads_daemon_threads The current number of live daemon threads
# TYPE jvm_threads_daemon_threads gauge
jvm_threads_daemon_threads{application="blade-admin",} 122.0
 
# HELP jvm_compilation_time_ms_total The approximate accumulated elapsed time spent in compilation
# TYPE jvm_compilation_time_ms_total counter
jvm_compilation_time_ms_total{application="blade-admin",compiler="HotSpot 64-Bit Tiered Compilers",} 19907.0
 
# HELP executor_pool_core_threads The core number of threads for the pool
# TYPE executor_pool_core_threads gauge
executor_pool_core_threads{application="blade-admin",name="applicationTaskExecutor",} 8.0
 
# HELP executor_pool_max_threads The maximum allowed number of threads in the pool
# TYPE executor_pool_max_threads gauge
executor_pool_max_threads{application="blade-admin",name="applicationTaskExecutor",} 2.147483647E9
 
# HELP jvm_memory_used_bytes The amount of used memory
# TYPE jvm_memory_used_bytes gauge
jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'profiled nmethods'",} 2.5495296E7
jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Survivor Space",} 5606000.0
jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Old Gen",} 3.5953152E7
jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'non-nmethods'",} 1376256.0
jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="Metaspace",} 7.1174928E7
jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Eden Space",} 0.0
jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'non-profiled nmethods'",} 6805248.0
jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="Compressed Class Space",} 9008928.0
 
# HELP jvm_gc_pause_seconds Time spent in GC pause
# TYPE jvm_gc_pause_seconds summary
jvm_gc_pause_seconds_count{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 1.0
jvm_gc_pause_seconds_sum{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 0.006
jvm_gc_pause_seconds_count{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 2.0
jvm_gc_pause_seconds_sum{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 0.031
 
# HELP jvm_gc_pause_seconds_max Time spent in GC pause
# TYPE jvm_gc_pause_seconds_max gauge
jvm_gc_pause_seconds_max{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 0.0
jvm_gc_pause_seconds_max{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 0.009
 
# HELP spring_security_filterchains_active_seconds_max
# TYPE spring_security_filterchains_active_seconds_max gauge
spring_security_filterchains_active_seconds_max{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
spring_security_filterchains_active_seconds_max{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0
 
# HELP spring_security_filterchains_active_seconds
# TYPE spring_security_filterchains_active_seconds summary
spring_security_filterchains_active_seconds_active_count{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
spring_security_filterchains_active_seconds_duration_sum{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
spring_security_filterchains_active_seconds_active_count{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0
spring_security_filterchains_active_seconds_duration_sum{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0

目前看到的完整结果如下

 HTTP/1.1 200 OK
Connection: close
Content-Type: text/plain;version=0.0.4;charset=utf-8
Content-Length: 6547
Date: Thu, 15 May 2025 08:39:07 GMT
 
# HELP undertow_connectors_bytes_sent_bytes
# TYPE undertow_connectors_bytes_sent_bytes gauge
undertow_connectors_bytes_sent_bytes{protocol="http",} 0.0
 
# HELP executor_pool_core_threads The core number of threads for the pool
# TYPE executor_pool_core_threads gauge
executor_pool_core_threads{name="applicationTaskExecutor",} 8.0
executor_pool_core_threads{name="taskScheduler",} 1.0
 
# HELP executor_queued_tasks The approximate number of tasks that are queued for execution
# TYPE executor_queued_tasks gauge
executor_queued_tasks{name="applicationTaskExecutor",} 0.0
executor_queued_tasks{name="taskScheduler",} 0.0
 
# HELP undertow_connectors_processing_time_max_ms
# TYPE undertow_connectors_processing_time_max_ms gauge
undertow_connectors_processing_time_max_ms{protocol="http",} 0.0

给业务服务加一个这个bean配置可开启jvm指标

 @Configuration
public class SysConfig {
 
    @Bean
    public InitializingBean forcePrometheusPostProcessor(BeanPostProcessor meterRegistryPostProcessor, PrometheusMeterRegistry registry) {
       return () -> meterRegistryPostProcessor.postProcessAfterInitialization(registry, "");
    }
 
}

我自己也找了一些信息，问题的原因是某些依赖的模块使用MeterRegistry的方式，导致MeterRegistry被过早初始化，此时MeterBinder还没有初始化完，错过了绑定的时机。解决方法就是在spring框架生命周期比较靠后的时机重新触发一次绑定，或者把要用的MeterBinder手工绑定一下。

使用ChatGPT对比了InitializingBean的方式和手工绑定的方式，ChatGPT更建议手工绑定的方式，理由是不破坏spring的生命周期，两种方式都能解决问题，手工绑定如下：

 @Bean
public ApplicationRunner bindJvmMetrics(MeterRegistry registry) {
    return args -> {
       try {
          // 手工new需要的MeterBinder并绑定
          new JvmThreadMetrics().bindTo(registry);
          new JvmCompilationMetrics().bindTo(registry);
          new JvmMemoryMetrics().bindTo(registry);
          new JvmGcMetrics().bindTo(registry);
          new JvmHeapPressureMetrics().bindTo(registry);
          new JvmInfoMetrics().bindTo(registry);
          new ProcessorMetrics().bindTo(registry);
          new ClassLoaderMetrics().bindTo(registry);
          new UptimeMetrics().bindTo(registry);
          new FileDescriptorMetrics().bindTo(registry);
          new LogbackMetrics().bindTo(registry);
       } catch (Exception e) {
          log.error(e.getMessage(), e);
       }
    };
}

	HTTP/1.1 200 OK
	Content-Type: text/plain;version=0.0.4;charset=utf-8
	Content-Length: 24963
	Cache-Control: no-cache, no-store, max-age=0, must-revalidate
	Pragma: no-cache
	Expires: 0
	X-Content-Type-Options: nosniff
	X-XSS-Protection: 0
	Referrer-Policy: no-referrer
	connection: close

	# HELP jvm_threads_daemon_threads The current number of live daemon threads
	# TYPE jvm_threads_daemon_threads gauge
	jvm_threads_daemon_threads{application="blade-admin",} 122.0

	# HELP jvm_compilation_time_ms_total The approximate accumulated elapsed time spent in compilation
	# TYPE jvm_compilation_time_ms_total counter
	jvm_compilation_time_ms_total{application="blade-admin",compiler="HotSpot 64-Bit Tiered Compilers",} 19907.0

	# HELP executor_pool_core_threads The core number of threads for the pool
	# TYPE executor_pool_core_threads gauge
	executor_pool_core_threads{application="blade-admin",name="applicationTaskExecutor",} 8.0

	# HELP executor_pool_max_threads The maximum allowed number of threads in the pool
	# TYPE executor_pool_max_threads gauge
	executor_pool_max_threads{application="blade-admin",name="applicationTaskExecutor",} 2.147483647E9

	# HELP jvm_memory_used_bytes The amount of used memory
	# TYPE jvm_memory_used_bytes gauge
	jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'profiled nmethods'",} 2.5495296E7
	jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Survivor Space",} 5606000.0
	jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Old Gen",} 3.5953152E7
	jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'non-nmethods'",} 1376256.0
	jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="Metaspace",} 7.1174928E7
	jvm_memory_used_bytes{application="blade-admin",area="heap",id="G1 Eden Space",} 0.0
	jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="CodeHeap 'non-profiled nmethods'",} 6805248.0
	jvm_memory_used_bytes{application="blade-admin",area="nonheap",id="Compressed Class Space",} 9008928.0

	# HELP jvm_gc_pause_seconds Time spent in GC pause
	# TYPE jvm_gc_pause_seconds summary
	jvm_gc_pause_seconds_count{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 1.0
	jvm_gc_pause_seconds_sum{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 0.006
	jvm_gc_pause_seconds_count{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 2.0
	jvm_gc_pause_seconds_sum{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 0.031

	# HELP jvm_gc_pause_seconds_max Time spent in GC pause
	# TYPE jvm_gc_pause_seconds_max gauge
	jvm_gc_pause_seconds_max{action="end of minor GC",application="blade-admin",cause="Metadata GC Threshold",gc="G1 Young Generation",} 0.0
	jvm_gc_pause_seconds_max{action="end of minor GC",application="blade-admin",cause="G1 Evacuation Pause",gc="G1 Young Generation",} 0.009

	# HELP spring_security_filterchains_active_seconds_max
	# TYPE spring_security_filterchains_active_seconds_max gauge
	spring_security_filterchains_active_seconds_max{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
	spring_security_filterchains_active_seconds_max{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0

	# HELP spring_security_filterchains_active_seconds
	# TYPE spring_security_filterchains_active_seconds summary
	spring_security_filterchains_active_seconds_active_count{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
	spring_security_filterchains_active_seconds_duration_sum{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="before",} 0.0
	spring_security_filterchains_active_seconds_active_count{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0
	spring_security_filterchains_active_seconds_duration_sum{application="blade-admin",spring_security_filterchain_position="0",spring_security_filterchain_size="0",spring_security_reached_filter_name="none",spring_security_reached_filter_section="after",} 0.0

	`HTTP/1.1 200 OK`
	Connection: close
	Content-Type: text/plain;version=0.0.4;charset=utf-8
	Content-Length: 6547
	Date: Thu, 15 May 2025 08:39:07 GMT

	# HELP undertow_connectors_bytes_sent_bytes
	# TYPE undertow_connectors_bytes_sent_bytes gauge
	undertow_connectors_bytes_sent_bytes{protocol="http",} 0.0

	# HELP executor_pool_core_threads The core number of threads for the pool
	# TYPE executor_pool_core_threads gauge
	executor_pool_core_threads{name="applicationTaskExecutor",} 8.0
	executor_pool_core_threads{name="taskScheduler",} 1.0

	# HELP executor_queued_tasks The approximate number of tasks that are queued for execution
	# TYPE executor_queued_tasks gauge
	executor_queued_tasks{name="applicationTaskExecutor",} 0.0
	executor_queued_tasks{name="taskScheduler",} 0.0

	# HELP undertow_connectors_processing_time_max_ms
	# TYPE undertow_connectors_processing_time_max_ms gauge
	undertow_connectors_processing_time_max_ms{protocol="http",} 0.0

	@Configuration
	public class SysConfig {

	@Bean
	public InitializingBean forcePrometheusPostProcessor(BeanPostProcessor meterRegistryPostProcessor, PrometheusMeterRegistry registry) {
	return () -> meterRegistryPostProcessor.postProcessAfterInitialization(registry, "");
	}

	}

	@Bean
	public ApplicationRunner bindJvmMetrics(MeterRegistry registry) {
	return args -> {
	try {
	// 手工new需要的MeterBinder并绑定
	new JvmThreadMetrics().bindTo(registry);
	new JvmCompilationMetrics().bindTo(registry);
	new JvmMemoryMetrics().bindTo(registry);
	new JvmGcMetrics().bindTo(registry);
	new JvmHeapPressureMetrics().bindTo(registry);
	new JvmInfoMetrics().bindTo(registry);
	new ProcessorMetrics().bindTo(registry);
	new ClassLoaderMetrics().bindTo(registry);
	new UptimeMetrics().bindTo(registry);
	new FileDescriptorMetrics().bindTo(registry);
	new LogbackMetrics().bindTo(registry);
	} catch (Exception e) {
	log.error(e.getMessage(), e);
	}
	};
	}