fix(hubble): clarify operations process metrics

hugegraph-hubble/README.md

@@ -689,7 +689,11 @@ its advertised origin and base path after normalizing scheme, hostname, and
 default port. A scheme-less advertised `host:port` uses the configured protocol.
 Existing node IDs remain unchanged; DNS aliases and ambiguous
 matches are not inferred. An unmatched URL reports `pd_metrics_unmapped` instead
-of assigning metrics to the leader. Store target discovery prefers
+of assigning metrics to the leader. Its separately labeled endpoint sample is one
+response from the configured URL, not a member identity or cluster aggregate; a
+load balancer may select a different member on each collection. Vermeer CPU
+display accepts lossless JSON decimal values while preserving unavailable and
+stale states. Store target discovery prefers
 `/v1/prom/targets/store`; only HTTP 404 or 405 falls back to
 `/v1/prom/targets-all`. Authentication, timeout, malformed-response, and server
 errors remain visible rather than triggering broad discovery.

hugegraph-hubble/docs/operations-diagnostics-20261023.md

@@ -95,10 +95,35 @@ uniquely identify the node serving CPU/JVM metrics. Hubble keeps the discovered
 PD topology and refuses to attach one member's system metrics to another member.
 The diagnostics panel compares the configured `pd.server` with advertised member
 REST addresses and separates that mapping issue from node health and readiness.
+When the endpoint cannot be uniquely matched, Hubble also requests its bounded
+`/actuator/prometheus` response once per metrics collection and shows the
+allowlisted CPU, memory and process values as a **configured endpoint sample**.
+This sample is not a sum and is never attached to a discovered member. A Service
+may route subsequent requests to different members, so changes between samples
+cannot be interpreted as one member's history. Failed or stale samples show their
+collection status instead of presenting old values as current. A topology-only
+request does not collect the sample.
 If member-specific system metrics are needed, review the intended routing before
 changing `pd.server` to a reachable member address (or correcting its `restUrl`),
 then restart Hubble. A scheme-less address uses `server.protocol`.
 
+Vermeer process CPU is an approximate display measurement. JSON responses are
+parsed with lossless numbers, so Hubble converts a finite nonnegative BigNumber
+to a JavaScript number at the resource display boundary. It does not convert
+graph identifiers or other precision-sensitive values. Missing, invalid and
+stale samples remain unavailable; zero is a valid CPU measurement. A low CPU
+sample can round to zero in the summary without becoming unavailable.
+
+Store node details also show process lifecycle when the Store supports
+`GET /-/lifecycle`. STARTING, STARTED, STOPPING and UNAVAILABLE describe the
+Store process and its gRPC server, not PD membership, partition health or
+read/write readiness. Hubble collects this separately from its existing
+readiness check, with bounded concurrency and the same collection deadline.
+An older Store returning HTTP 404 is shown as unsupported; a failed or stale
+observation is not presented as a current process state. The endpoint is
+introduced by [Store lifecycle support](https://github.com/hugegraph/hugegraph/pull/362)
+and requires a Store version containing that change.
+
 For Store Raft counters, Hubble first requests `/metrics/raft/summary`, whose
 response is `{"groups":32,"enabled_groups":32}`. The enabled count means JRaft
 metrics collection is enabled, not that the groups are healthy. HTTP 404/405

hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/DefaultOperationsDataService.java

@@ -411,15 +411,18 @@ private static Node staleNode(Node node, SourceStatus source) {
             statuses.put(group, staleStatus);
         });
         return copyNode(node, node.getMetrics(), statuses).withReadiness(
-                node.getReadiness() == null ? null : node.getReadiness().asStale());
+                node.getReadiness() == null ? null : node.getReadiness().asStale())
+                .withProcessLifecycle(node.getProcessLifecycle() == null ? null :
+                                      node.getProcessLifecycle().asStale());
     }
 
     private static Node copyNode(Node node, Map<String, Object> metrics,
                                  Map<String, MetricStatus> statuses) {
         return new Node(node.getId(), node.getType(), node.getName(),
                         node.getRole(), node.getVersion(), node.getStatus(),
                         node.getObservedAt(), metrics, statuses)
-                        .withReadiness(node.getReadiness());
+                        .withReadiness(node.getReadiness())
+                        .withProcessLifecycle(node.getProcessLifecycle());
     }
 
     private static boolean failed(SourceStatus source) {

hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/LiveOperationsCollector.java

@@ -55,6 +55,7 @@
 import org.apache.hugegraph.service.op.OperationsModels.MetricStatus;
 import org.apache.hugegraph.service.op.OperationsModels.Snapshot;
 import org.apache.hugegraph.service.op.OperationsModels.SourceStatus;
+import org.apache.hugegraph.service.op.OperationsModels.Lifecycle;
 import org.apache.hugegraph.service.op.OperationsModels.Topology;
 
 @Service
@@ -80,6 +81,7 @@ public class LiveOperationsCollector implements OperationsCollector {
     private final Clock clock;
     private final ExecutorService storeExecutor;
     private final ExecutorService readinessExecutor;
+    private final ExecutorService lifecycleExecutor;
     private final int storeDeadlineMillis;
     private final Set<String> storeAllowedTargets;
     private final ServerClientProvider serverClients;
@@ -180,6 +182,14 @@ public LiveOperationsCollector(HugeConfig config, ObjectMapper mapper,
                     thread.setDaemon(true);
                     return thread;
                 });
+        // Lifecycle is optional and must not consume readiness workers.
+        this.lifecycleExecutor = Executors.newFixedThreadPool(
+                storeThreads, runnable -> {
+                    Thread thread = new Thread(
+                            runnable, "hubble-store-lifecycle-" + THREAD_SEQUENCE.incrementAndGet());
+                    thread.setDaemon(true);
+                    return thread;
+                });
         this.storeExecutor = Executors.newFixedThreadPool(
                 storeThreads, runnable -> {
                     Thread thread = new Thread(
@@ -194,9 +204,11 @@ public LiveOperationsCollector(HugeConfig config, ObjectMapper mapper,
     public void close() {
         this.storeExecutor.shutdownNow();
         this.readinessExecutor.shutdownNow();
+        this.lifecycleExecutor.shutdownNow();
         try {
             this.storeExecutor.awaitTermination(1L, TimeUnit.SECONDS);
             this.readinessExecutor.awaitTermination(1L, TimeUnit.SECONDS);
+            this.lifecycleExecutor.awaitTermination(1L, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         }
@@ -651,6 +663,22 @@ private SourceStatus collectPdMetrics(long now, SourceStatus status,
             details.put("affected_node_count", 1);
             details.put("failed_metric_groups", List.of("system"));
             details.put("pd_metric_mapping", this.parser.pdMetricMappingDiagnostics(cluster, this.pdBase));
+            // A Service or proxy may route to a different member on each
+            // request. Preserve its bounded response as an endpoint sample,
+            // never as a member metric or a cluster aggregate.
+            Map<String, Object> endpoint = new LinkedHashMap<>();
+            try {
+                endpoint.put("metrics", this.parser.parsePdPrometheusMetrics(
+                        this.http.get(URI.create(this.pdBase + "/actuator/prometheus"),
+                                      this.pdUsername, this.pdPassword,
+                                      Collections.emptySet(), "text/plain")));
+                endpoint.put("metric_status", availableMetric(now));
+            } catch (RuntimeException e) {
+                Map<String, Object> failure = new LinkedHashMap<>(diagnostics(e));
+                failure.putIfAbsent("request_path", "/actuator/prometheus");
+                endpoint.put("metric_status", metricStatus(e, now, true).withDiagnostics(failure));
+            }
+            details.put("pd_endpoint_metrics", endpoint);
             return new SourceStatus("PARTIAL", status.getStatus(), now,
                                     status.getLastSuccessAt(), false, false,
                                     preferredReason(status,
@@ -725,6 +753,7 @@ private SourceStatus collectStoreMetrics(String stores, long now,
                                              SourceStatus status,
                                              List<Node> nodes) {
         List<Future<Readiness>> readiness = new ArrayList<>();
+        List<Future<Lifecycle>> lifecycles = new ArrayList<>();
         try {
             Map<String, String> restAddresses;
             try {
@@ -796,6 +825,8 @@ private SourceStatus collectStoreMetrics(String stores, long now,
                 readiness.add(this.readinessExecutor.submit(() -> this.probeReadiness(
                         "STORE", job.getTarget().resolve("/-/state"),
                         this.storeUsername, this.storePassword, allowedTargets, now)));
+                lifecycles.add(this.lifecycleExecutor.submit(() -> this.probeLifecycle(
+                        job.getTarget().resolve("/-/lifecycle"), allowedTargets, now)));
             }
             List<Future<StoreMetricResult>> futures = this.storeExecutor.invokeAll(
                     jobs.stream().map(job -> (java.util.concurrent.Callable<
@@ -830,6 +861,11 @@ private SourceStatus collectStoreMetrics(String stores, long now,
                         readiness.get(i), job, now, readinessDeadline);
                 nodes.set(job.getNodeIndex(), nodes.get(job.getNodeIndex()).withReadiness(observation));
             }
+            for (int i = 0; i < jobs.size(); i++) {
+                StoreMetricJob job = jobs.get(i);
+                Lifecycle observation = this.lifecycleResult(lifecycles.get(i), now, readinessDeadline);
+                nodes.set(job.getNodeIndex(), nodes.get(job.getNodeIndex()).withProcessLifecycle(observation));
+            }
             if (partial) {
                 return metricFailureStatus(status, now, successfulGroups, failureReason)
                        .withDiagnostics(storeFailureSummary(nodes));
@@ -847,6 +883,28 @@ private SourceStatus collectStoreMetrics(String stores, long now,
                    .withDiagnostics(diagnostics(e));
         } finally {
             readiness.forEach(future -> future.cancel(true));
+            lifecycles.forEach(future -> future.cancel(true));
+        }
+    }
+
+    private Lifecycle probeLifecycle(URI target, Set<String> allowedTargets, long now) {
+        try {
+            OperationsHttpClient.ProbeResponse response = this.http.probe(
+                    target, this.storeUsername, this.storePassword, allowedTargets);
+            return this.parser.parseStoreLifecycle(response.getBody(), response.getStatus(), now);
+        } catch (RuntimeException e) {
+            return new Lifecycle("UNKNOWN", now, metricReason(e), false);
+        }
+    }
+
+    private Lifecycle lifecycleResult(Future<Lifecycle> future, long now, long deadline)
+                                      throws InterruptedException {
+        try {
+            return future.get(Math.max(0L, deadline - System.nanoTime()), TimeUnit.NANOSECONDS);
+        } catch (CancellationException | TimeoutException e) {
+            return new Lifecycle("UNKNOWN", now, "upstream_deadline", false);
+        } catch (ExecutionException e) {
+            return new Lifecycle("UNKNOWN", now, "upstream_unavailable", false);
         }
     }
 
@@ -1064,7 +1122,8 @@ private static Node copyNode(Node node, Map<String, Object> metrics,
         return new Node(node.getId(), node.getType(), node.getName(),
                         node.getRole(), node.getVersion(), node.getStatus(),
                         node.getObservedAt(), metrics, statuses)
-                        .withReadiness(node.getReadiness());
+                        .withReadiness(node.getReadiness())
+                        .withProcessLifecycle(node.getProcessLifecycle());
     }
 
     private static SourceStatus withStatus(SourceStatus source,

hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsModels.java

@@ -45,6 +45,7 @@ public static final class Node {
         private final Map<String, Object> metrics;
         private final Map<String, MetricStatus> metricStatuses;
         private final Readiness readiness;
+        private final Lifecycle processLifecycle;
 
         public Node(String id, String type, String name, String role,
                     String version, String status, Long observedAt,
@@ -58,14 +59,14 @@ public Node(String id, String type, String name, String role,
                     Map<String, Object> metrics,
                     Map<String, MetricStatus> metricStatuses) {
             this(id, type, name, role, version, status, observedAt, metrics,
-                 metricStatuses, null);
+                 metricStatuses, null, null);
         }
 
         private Node(String id, String type, String name, String role,
                      String version, String status, Long observedAt,
                      Map<String, Object> metrics,
                      Map<String, MetricStatus> metricStatuses,
-                     Readiness readiness) {
+                     Readiness readiness, Lifecycle processLifecycle) {
             this.id = id;
             this.type = type;
             this.name = name;
@@ -76,6 +77,7 @@ private Node(String id, String type, String name, String role,
             this.metrics = immutableMap(metrics);
             this.metricStatuses = immutableMap(metricStatuses);
             this.readiness = readiness;
+            this.processLifecycle = processLifecycle;
         }
 
         public String getId() {
@@ -121,7 +123,53 @@ public Readiness getReadiness() {
         public Node withReadiness(Readiness value) {
             return new Node(this.id, this.type, this.name, this.role,
                             this.version, this.status, this.observedAt,
-                            this.metrics, this.metricStatuses, value);
+                            this.metrics, this.metricStatuses, value, this.processLifecycle);
+        }
+
+        public Lifecycle getProcessLifecycle() {
+            return this.processLifecycle;
+        }
+
+        public Node withProcessLifecycle(Lifecycle value) {
+            return new Node(this.id, this.type, this.name, this.role,
+                            this.version, this.status, this.observedAt,
+                            this.metrics, this.metricStatuses, this.readiness, value);
+        }
+    }
+
+    // Process lifecycle is independent of partition and read/write readiness.
+    @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+    public static final class Lifecycle {
+        private final String state;
+        private final Long observedAt;
+        private final String reason;
+        private final boolean stale;
+
+        public Lifecycle(String state, Long observedAt, String reason, boolean stale) {
+            this.state = state;
+            this.observedAt = observedAt;
+            this.reason = reason;
+            this.stale = stale;
+        }
+
+        public String getState() {
+            return this.state;
+        }
+
+        public Long getObservedAt() {
+            return this.observedAt;
+        }
+
+        public String getReason() {
+            return this.reason;
+        }
+
+        public boolean isStale() {
+            return this.stale;
+        }
+
+        public Lifecycle asStale() {
+            return new Lifecycle(this.state, this.observedAt, this.reason, true);
         }
     }
 
@@ -480,7 +528,9 @@ public Snapshot stale(String staleReason) {
                                node.getRole(), node.getVersion(), node.getStatus(),
                                node.getObservedAt(), node.getMetrics(), statuses)
                                .withReadiness(node.getReadiness() == null ? null :
-                                              node.getReadiness().asStale()));
+                                              node.getReadiness().asStale())
+                               .withProcessLifecycle(node.getProcessLifecycle() == null ? null :
+                                                     node.getProcessLifecycle().asStale()));
             }
             return new Snapshot("DEGRADED", this.observedAt, true, staleReason,
                                 staleSources, staleNodes, this.facts);

hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsPayloadParser.java

@@ -34,6 +34,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.apache.hugegraph.service.op.OperationsModels.Node;
+import org.apache.hugegraph.service.op.OperationsModels.Lifecycle;
 import org.apache.hugegraph.service.op.OperationsModels.Readiness;
 import org.apache.hugegraph.service.op.OperationsModels.MetricStatus;
 import org.apache.hugegraph.service.op.OperationsModels.Topology;
@@ -46,6 +47,22 @@ public OperationsPayloadParser(ObjectMapper mapper) {
         this.mapper = mapper;
     }
 
+    Lifecycle parseStoreLifecycle(String payload, int httpStatus, long observedAt) {
+        try {
+            JsonNode root = this.mapper.readTree(payload);
+            JsonNode code = root == null ? null : root.get("status");
+            String state = root == null ? null : this.text(root, "text");
+            if (httpStatus != 200 || code == null || !code.isIntegralNumber() ||
+                !code.canConvertToInt() || code.intValue() != 0 ||
+                !List.of("STARTING", "STARTED", "STOPPING", "UNAVAILABLE").contains(state == null ? "" : state)) {
+                throw new MalformedUpstreamException("store_lifecycle");
+            }
+            return new Lifecycle(state, observedAt, null, false);
+        } catch (Exception e) {
+            return new Lifecycle("UNKNOWN", observedAt, "response_unrecognized", false);
+        }
+    }
+
     Readiness parseReadiness(String type, String payload, int httpStatus,
                               long observedAt, String requestPath) {
         Map<String, Object> details = Map.of("request_path", requestPath,