diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index e878649f2..e2d1eef3d 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -689,7 +689,11 @@ its advertised origin and base path after normalizing scheme, hostname, and default port. A scheme-less advertised `host:port` uses the configured protocol. Existing node IDs remain unchanged; DNS aliases and ambiguous matches are not inferred. An unmatched URL reports `pd_metrics_unmapped` instead -of assigning metrics to the leader. Store target discovery prefers +of assigning metrics to the leader. Its separately labeled endpoint sample is one +response from the configured URL, not a member identity or cluster aggregate; a +load balancer may select a different member on each collection. Vermeer CPU +display accepts lossless JSON decimal values while preserving unavailable and +stale states. Store target discovery prefers `/v1/prom/targets/store`; only HTTP 404 or 405 falls back to `/v1/prom/targets-all`. Authentication, timeout, malformed-response, and server errors remain visible rather than triggering broad discovery. diff --git a/hugegraph-hubble/docs/operations-diagnostics-20261023.md b/hugegraph-hubble/docs/operations-diagnostics-20261023.md index 702e84cac..ccf898446 100644 --- a/hugegraph-hubble/docs/operations-diagnostics-20261023.md +++ b/hugegraph-hubble/docs/operations-diagnostics-20261023.md @@ -95,10 +95,35 @@ uniquely identify the node serving CPU/JVM metrics. Hubble keeps the discovered PD topology and refuses to attach one member's system metrics to another member. The diagnostics panel compares the configured `pd.server` with advertised member REST addresses and separates that mapping issue from node health and readiness. +When the endpoint cannot be uniquely matched, Hubble also requests its bounded +`/actuator/prometheus` response once per metrics collection and shows the +allowlisted CPU, memory and process values as a **configured endpoint sample**. +This sample is not a sum and is never attached to a discovered member. A Service +may route subsequent requests to different members, so changes between samples +cannot be interpreted as one member's history. Failed or stale samples show their +collection status instead of presenting old values as current. A topology-only +request does not collect the sample. If member-specific system metrics are needed, review the intended routing before changing `pd.server` to a reachable member address (or correcting its `restUrl`), then restart Hubble. A scheme-less address uses `server.protocol`. +Vermeer process CPU is an approximate display measurement. JSON responses are +parsed with lossless numbers, so Hubble converts a finite nonnegative BigNumber +to a JavaScript number at the resource display boundary. It does not convert +graph identifiers or other precision-sensitive values. Missing, invalid and +stale samples remain unavailable; zero is a valid CPU measurement. A low CPU +sample can round to zero in the summary without becoming unavailable. + +Store node details also show process lifecycle when the Store supports +`GET /-/lifecycle`. STARTING, STARTED, STOPPING and UNAVAILABLE describe the +Store process and its gRPC server, not PD membership, partition health or +read/write readiness. Hubble collects this separately from its existing +readiness check, with bounded concurrency and the same collection deadline. +An older Store returning HTTP 404 is shown as unsupported; a failed or stale +observation is not presented as a current process state. The endpoint is +introduced by [Store lifecycle support](https://github.com/hugegraph/hugegraph/pull/362) +and requires a Store version containing that change. + For Store Raft counters, Hubble first requests `/metrics/raft/summary`, whose response is `{"groups":32,"enabled_groups":32}`. The enabled count means JRaft metrics collection is enabled, not that the groups are healthy. HTTP 404/405 diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/DefaultOperationsDataService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/DefaultOperationsDataService.java index 762c165f2..b77ab51e5 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/DefaultOperationsDataService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/DefaultOperationsDataService.java @@ -411,7 +411,9 @@ private static Node staleNode(Node node, SourceStatus source) { statuses.put(group, staleStatus); }); return copyNode(node, node.getMetrics(), statuses).withReadiness( - node.getReadiness() == null ? null : node.getReadiness().asStale()); + node.getReadiness() == null ? null : node.getReadiness().asStale()) + .withProcessLifecycle(node.getProcessLifecycle() == null ? null : + node.getProcessLifecycle().asStale()); } private static Node copyNode(Node node, Map metrics, @@ -419,7 +421,8 @@ private static Node copyNode(Node node, Map metrics, return new Node(node.getId(), node.getType(), node.getName(), node.getRole(), node.getVersion(), node.getStatus(), node.getObservedAt(), metrics, statuses) - .withReadiness(node.getReadiness()); + .withReadiness(node.getReadiness()) + .withProcessLifecycle(node.getProcessLifecycle()); } private static boolean failed(SourceStatus source) { diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/LiveOperationsCollector.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/LiveOperationsCollector.java index 8cf4d6906..d34badeb8 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/LiveOperationsCollector.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/LiveOperationsCollector.java @@ -55,6 +55,7 @@ import org.apache.hugegraph.service.op.OperationsModels.MetricStatus; import org.apache.hugegraph.service.op.OperationsModels.Snapshot; import org.apache.hugegraph.service.op.OperationsModels.SourceStatus; +import org.apache.hugegraph.service.op.OperationsModels.Lifecycle; import org.apache.hugegraph.service.op.OperationsModels.Topology; @Service @@ -80,6 +81,7 @@ public class LiveOperationsCollector implements OperationsCollector { private final Clock clock; private final ExecutorService storeExecutor; private final ExecutorService readinessExecutor; + private final ExecutorService lifecycleExecutor; private final int storeDeadlineMillis; private final Set storeAllowedTargets; private final ServerClientProvider serverClients; @@ -180,6 +182,14 @@ public LiveOperationsCollector(HugeConfig config, ObjectMapper mapper, thread.setDaemon(true); return thread; }); + // Lifecycle is optional and must not consume readiness workers. + this.lifecycleExecutor = Executors.newFixedThreadPool( + storeThreads, runnable -> { + Thread thread = new Thread( + runnable, "hubble-store-lifecycle-" + THREAD_SEQUENCE.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); this.storeExecutor = Executors.newFixedThreadPool( storeThreads, runnable -> { Thread thread = new Thread( @@ -194,9 +204,11 @@ public LiveOperationsCollector(HugeConfig config, ObjectMapper mapper, public void close() { this.storeExecutor.shutdownNow(); this.readinessExecutor.shutdownNow(); + this.lifecycleExecutor.shutdownNow(); try { this.storeExecutor.awaitTermination(1L, TimeUnit.SECONDS); this.readinessExecutor.awaitTermination(1L, TimeUnit.SECONDS); + this.lifecycleExecutor.awaitTermination(1L, TimeUnit.SECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -651,6 +663,22 @@ private SourceStatus collectPdMetrics(long now, SourceStatus status, details.put("affected_node_count", 1); details.put("failed_metric_groups", List.of("system")); details.put("pd_metric_mapping", this.parser.pdMetricMappingDiagnostics(cluster, this.pdBase)); + // A Service or proxy may route to a different member on each + // request. Preserve its bounded response as an endpoint sample, + // never as a member metric or a cluster aggregate. + Map endpoint = new LinkedHashMap<>(); + try { + endpoint.put("metrics", this.parser.parsePdPrometheusMetrics( + this.http.get(URI.create(this.pdBase + "/actuator/prometheus"), + this.pdUsername, this.pdPassword, + Collections.emptySet(), "text/plain"))); + endpoint.put("metric_status", availableMetric(now)); + } catch (RuntimeException e) { + Map failure = new LinkedHashMap<>(diagnostics(e)); + failure.putIfAbsent("request_path", "/actuator/prometheus"); + endpoint.put("metric_status", metricStatus(e, now, true).withDiagnostics(failure)); + } + details.put("pd_endpoint_metrics", endpoint); return new SourceStatus("PARTIAL", status.getStatus(), now, status.getLastSuccessAt(), false, false, preferredReason(status, @@ -725,6 +753,7 @@ private SourceStatus collectStoreMetrics(String stores, long now, SourceStatus status, List nodes) { List> readiness = new ArrayList<>(); + List> lifecycles = new ArrayList<>(); try { Map restAddresses; try { @@ -796,6 +825,8 @@ private SourceStatus collectStoreMetrics(String stores, long now, readiness.add(this.readinessExecutor.submit(() -> this.probeReadiness( "STORE", job.getTarget().resolve("/-/state"), this.storeUsername, this.storePassword, allowedTargets, now))); + lifecycles.add(this.lifecycleExecutor.submit(() -> this.probeLifecycle( + job.getTarget().resolve("/-/lifecycle"), allowedTargets, now))); } List> futures = this.storeExecutor.invokeAll( jobs.stream().map(job -> (java.util.concurrent.Callable< @@ -830,6 +861,11 @@ private SourceStatus collectStoreMetrics(String stores, long now, readiness.get(i), job, now, readinessDeadline); nodes.set(job.getNodeIndex(), nodes.get(job.getNodeIndex()).withReadiness(observation)); } + for (int i = 0; i < jobs.size(); i++) { + StoreMetricJob job = jobs.get(i); + Lifecycle observation = this.lifecycleResult(lifecycles.get(i), now, readinessDeadline); + nodes.set(job.getNodeIndex(), nodes.get(job.getNodeIndex()).withProcessLifecycle(observation)); + } if (partial) { return metricFailureStatus(status, now, successfulGroups, failureReason) .withDiagnostics(storeFailureSummary(nodes)); @@ -847,6 +883,28 @@ private SourceStatus collectStoreMetrics(String stores, long now, .withDiagnostics(diagnostics(e)); } finally { readiness.forEach(future -> future.cancel(true)); + lifecycles.forEach(future -> future.cancel(true)); + } + } + + private Lifecycle probeLifecycle(URI target, Set allowedTargets, long now) { + try { + OperationsHttpClient.ProbeResponse response = this.http.probe( + target, this.storeUsername, this.storePassword, allowedTargets); + return this.parser.parseStoreLifecycle(response.getBody(), response.getStatus(), now); + } catch (RuntimeException e) { + return new Lifecycle("UNKNOWN", now, metricReason(e), false); + } + } + + private Lifecycle lifecycleResult(Future future, long now, long deadline) + throws InterruptedException { + try { + return future.get(Math.max(0L, deadline - System.nanoTime()), TimeUnit.NANOSECONDS); + } catch (CancellationException | TimeoutException e) { + return new Lifecycle("UNKNOWN", now, "upstream_deadline", false); + } catch (ExecutionException e) { + return new Lifecycle("UNKNOWN", now, "upstream_unavailable", false); } } @@ -1064,7 +1122,8 @@ private static Node copyNode(Node node, Map metrics, return new Node(node.getId(), node.getType(), node.getName(), node.getRole(), node.getVersion(), node.getStatus(), node.getObservedAt(), metrics, statuses) - .withReadiness(node.getReadiness()); + .withReadiness(node.getReadiness()) + .withProcessLifecycle(node.getProcessLifecycle()); } private static SourceStatus withStatus(SourceStatus source, diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsModels.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsModels.java index eafabe211..83eb189de 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsModels.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsModels.java @@ -45,6 +45,7 @@ public static final class Node { private final Map metrics; private final Map metricStatuses; private final Readiness readiness; + private final Lifecycle processLifecycle; public Node(String id, String type, String name, String role, String version, String status, Long observedAt, @@ -58,14 +59,14 @@ public Node(String id, String type, String name, String role, Map metrics, Map metricStatuses) { this(id, type, name, role, version, status, observedAt, metrics, - metricStatuses, null); + metricStatuses, null, null); } private Node(String id, String type, String name, String role, String version, String status, Long observedAt, Map metrics, Map metricStatuses, - Readiness readiness) { + Readiness readiness, Lifecycle processLifecycle) { this.id = id; this.type = type; this.name = name; @@ -76,6 +77,7 @@ private Node(String id, String type, String name, String role, this.metrics = immutableMap(metrics); this.metricStatuses = immutableMap(metricStatuses); this.readiness = readiness; + this.processLifecycle = processLifecycle; } public String getId() { @@ -121,7 +123,53 @@ public Readiness getReadiness() { public Node withReadiness(Readiness value) { return new Node(this.id, this.type, this.name, this.role, this.version, this.status, this.observedAt, - this.metrics, this.metricStatuses, value); + this.metrics, this.metricStatuses, value, this.processLifecycle); + } + + public Lifecycle getProcessLifecycle() { + return this.processLifecycle; + } + + public Node withProcessLifecycle(Lifecycle value) { + return new Node(this.id, this.type, this.name, this.role, + this.version, this.status, this.observedAt, + this.metrics, this.metricStatuses, this.readiness, value); + } + } + + // Process lifecycle is independent of partition and read/write readiness. + @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) + public static final class Lifecycle { + private final String state; + private final Long observedAt; + private final String reason; + private final boolean stale; + + public Lifecycle(String state, Long observedAt, String reason, boolean stale) { + this.state = state; + this.observedAt = observedAt; + this.reason = reason; + this.stale = stale; + } + + public String getState() { + return this.state; + } + + public Long getObservedAt() { + return this.observedAt; + } + + public String getReason() { + return this.reason; + } + + public boolean isStale() { + return this.stale; + } + + public Lifecycle asStale() { + return new Lifecycle(this.state, this.observedAt, this.reason, true); } } @@ -480,7 +528,9 @@ public Snapshot stale(String staleReason) { node.getRole(), node.getVersion(), node.getStatus(), node.getObservedAt(), node.getMetrics(), statuses) .withReadiness(node.getReadiness() == null ? null : - node.getReadiness().asStale())); + node.getReadiness().asStale()) + .withProcessLifecycle(node.getProcessLifecycle() == null ? null : + node.getProcessLifecycle().asStale())); } return new Snapshot("DEGRADED", this.observedAt, true, staleReason, staleSources, staleNodes, this.facts); diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsPayloadParser.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsPayloadParser.java index 671f03961..218cc819e 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsPayloadParser.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/op/OperationsPayloadParser.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hugegraph.service.op.OperationsModels.Node; +import org.apache.hugegraph.service.op.OperationsModels.Lifecycle; import org.apache.hugegraph.service.op.OperationsModels.Readiness; import org.apache.hugegraph.service.op.OperationsModels.MetricStatus; import org.apache.hugegraph.service.op.OperationsModels.Topology; @@ -46,6 +47,22 @@ public OperationsPayloadParser(ObjectMapper mapper) { this.mapper = mapper; } + Lifecycle parseStoreLifecycle(String payload, int httpStatus, long observedAt) { + try { + JsonNode root = this.mapper.readTree(payload); + JsonNode code = root == null ? null : root.get("status"); + String state = root == null ? null : this.text(root, "text"); + if (httpStatus != 200 || code == null || !code.isIntegralNumber() || + !code.canConvertToInt() || code.intValue() != 0 || + !List.of("STARTING", "STARTED", "STOPPING", "UNAVAILABLE").contains(state == null ? "" : state)) { + throw new MalformedUpstreamException("store_lifecycle"); + } + return new Lifecycle(state, observedAt, null, false); + } catch (Exception e) { + return new Lifecycle("UNKNOWN", observedAt, "response_unrecognized", false); + } + } + Readiness parseReadiness(String type, String payload, int httpStatus, long observedAt, String requestPath) { Map details = Map.of("request_path", requestPath, diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/LiveOperationsCollectorTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/LiveOperationsCollectorTest.java index f91eb441c..0889de8ed 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/LiveOperationsCollectorTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/LiveOperationsCollectorTest.java @@ -639,7 +639,7 @@ public void testUnmatchedPdAddressNeverAttributesMetricsToLeader() pd.stop(0); } - Assert.assertEquals(0, probes.get()); + Assert.assertEquals(1, probes.get()); Assert.assertEquals("PARTIAL", snapshot.getSources().get("pd").getAvailability()); Assert.assertEquals("pd_metrics_unmapped", snapshot.getSources().get("pd").getReason()); Assert.assertEquals("UP", snapshot.getSources().get("pd").getStatus()); @@ -647,6 +647,12 @@ public void testUnmatchedPdAddressNeverAttributesMetricsToLeader() .get("affected_node_count")); Assert.assertEquals(List.of("system"), snapshot.getSources().get("pd").getDiagnostics() .get("failed_metric_groups")); + Map endpoint = (Map) snapshot.getSources().get("pd").getDiagnostics() + .get("pd_endpoint_metrics"); + MetricStatus endpointStatus = (MetricStatus) endpoint.get("metric_status"); + Assert.assertEquals("UNAVAILABLE", endpointStatus.getAvailability()); + Assert.assertEquals("/actuator/prometheus", endpointStatus.getDiagnostics().get("request_path")); + Assert.assertFalse(endpoint.containsKey("metrics")); OperationsModels.Node leader = snapshot.getNodes().stream() .filter(node -> "PD".equals(node.getType())) .findFirst().orElseThrow(AssertionError::new); @@ -656,6 +662,94 @@ public void testUnmatchedPdAddressNeverAttributesMetricsToLeader() Assert.assertEquals("metrics_not_collected", leader.getMetricStatuses().get("system").getReason()); } + @Test + public void testUnmatchedPdEndpointSampleIsNeverAssignedToMembers() throws IOException { + String cluster = new ObjectMapper().writeValueAsString(Map.of("status", 0, + "data", Map.of("state", "Cluster_OK", "pdList", List.of( + Map.of("restUrl", "http://pd-a:8620", "state", "Up", "role", "Leader"), + Map.of("restUrl", "http://pd-b:8620", "state", "Up", "role", "Follower"))))); + HttpServer pd = pdServer(200, cluster, 200, stores()); + AtomicInteger probes = new AtomicInteger(); + pd.removeContext("/actuator/prometheus"); + pd.createContext("/actuator/prometheus", exchange -> { + probes.incrementAndGet(); + byte[] bytes = ("process_cpu_usage 0.125\njvm_threads_live_threads 8\n" + + "untrusted_private_metric 42\n").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + exchange.getResponseBody().write(bytes); + exchange.close(); + }); + Snapshot snapshot; + try { + snapshot = collector(true, pd).collect(serverClient(), true); + } finally { + pd.stop(0); + } + Assert.assertEquals(1, probes.get()); + Assert.assertEquals("PARTIAL", snapshot.getSources().get("pd").getAvailability()); + Assert.assertEquals("pd_metrics_unmapped", snapshot.getSources().get("pd").getReason()); + Map endpoint = (Map) snapshot.getSources().get("pd").getDiagnostics() + .get("pd_endpoint_metrics"); + Map metrics = (Map) endpoint.get("metrics"); + Assert.assertEquals(0.125D, metrics.get("process_cpu_usage")); + Assert.assertEquals(8D, metrics.get("threads_live")); + Assert.assertFalse(metrics.containsKey("untrusted_private_metric")); + Assert.assertEquals("AVAILABLE", ((MetricStatus) endpoint.get("metric_status")).getAvailability()); + for (OperationsModels.Node node : snapshot.getNodes()) { + if ("PD".equals(node.getType())) { + Assert.assertFalse(node.getMetrics().containsKey("system")); + Assert.assertEquals("metrics_not_collected", node.getMetricStatuses().get("system").getReason()); + } + } + } + + @Test + public void testUnmatchedPdEndpointSampleIsNotCollectedForTopologyOnly() throws IOException { + HttpServer pd = pdServer(200, cluster().replace("http://pd:8620", "http://pd-alias:8620"), + 200, stores()); + AtomicInteger probes = new AtomicInteger(); + pd.removeContext("/actuator/prometheus"); + pd.createContext("/actuator/prometheus", exchange -> { + probes.incrementAndGet(); + exchange.sendResponseHeaders(500, -1); + exchange.close(); + }); + Snapshot snapshot; + try { + snapshot = collector(true, pd).collect(serverClient(), false); + } finally { + pd.stop(0); + } + Assert.assertEquals(0, probes.get()); + Assert.assertFalse(snapshot.getSources().get("pd").getDiagnostics().containsKey("pd_endpoint_metrics")); + } + + @Test + public void testMalformedUnmatchedPdSampleDoesNotExposePayload() throws IOException { + HttpServer pd = pdServer(200, cluster().replace("http://pd:8620", "http://pd-alias:8620"), + 200, stores()); + pd.removeContext("/actuator/prometheus"); + pd.createContext("/actuator/prometheus", exchange -> { + byte[] bytes = "untrusted_secret_metric 99\n".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + exchange.getResponseBody().write(bytes); + exchange.close(); + }); + Snapshot snapshot; + try { + snapshot = collector(true, pd).collect(serverClient(), true); + } finally { + pd.stop(0); + } + Map endpoint = (Map) snapshot.getSources().get("pd").getDiagnostics() + .get("pd_endpoint_metrics"); + MetricStatus result = (MetricStatus) endpoint.get("metric_status"); + Assert.assertEquals("MALFORMED", result.getAvailability()); + Assert.assertFalse(endpoint.containsKey("metrics")); + Assert.assertFalse(new ObjectMapper().writeValueAsString(snapshot).contains("untrusted_secret_metric")); + Assert.assertEquals("pd_metrics_unmapped", snapshot.getSources().get("pd").getReason()); + } + @Test public void testUnmatchedPdMetricsDoesNotAttributeFailureToDiscoveredNodes() throws IOException { @@ -1680,6 +1774,66 @@ public void testStoreStartingAndOnlineDoNotRewritePdRegistration() throws Except } } + @Test + public void testStoreLifecycleIsIndependentOfReadinessAndOldStore404() throws Exception { + for (String state : new String[]{"STARTING", "STARTED", "STOPPING", "UNAVAILABLE", "404"}) { + HttpServer pd = pdServer(200, cluster(), 200, stores()); + context(pd, "/v1/ready", 200, "{\"ready\":true,\"state\":\"STATE_FOLLOWER\"}"); + context(pd, "/-/state", 200, "{\"status\":0,\"text\":\"STARTING\"}"); + if (!"404".equals(state)) { + context(pd, "/-/lifecycle", 200, "{\"status\":0,\"text\":\"" + state + "\"}"); + } + context(pd, "/v1/prom/targets/store", 200, targets("127.0.0.1:" + pd.getAddress().getPort())); + storeMetricContexts(pd); + LiveOperationsCollector collector = collector(true, pd); + try { + OperationsModels.Node node = store(collector.collect(serverClient(), true)); + Assert.assertEquals("UP", node.getStatus()); + Assert.assertEquals("NOT_READY", node.getReadiness().getStatus()); + Assert.assertEquals("STARTING", node.getReadiness().getState()); + Assert.assertEquals("AVAILABLE", node.getMetricStatuses().get("system").getAvailability()); + Assert.assertEquals("404".equals(state) ? "UNKNOWN" : state, + node.getProcessLifecycle().getState()); + if ("404".equals(state)) { + Assert.assertEquals("unsupported_version", node.getProcessLifecycle().getReason()); + } + } finally { + collector.close(); + pd.stop(0); + } + } + } + + @Test + public void testSlowLifecycleDoesNotConsumeReadinessOrMetricBudget() throws Exception { + java.util.concurrent.ExecutorService upstream = java.util.concurrent.Executors.newCachedThreadPool(); + HttpServer pd = pdServer(200, cluster(), 200, stores(), upstream); + context(pd, "/v1/ready", 200, "{\"ready\":true,\"state\":\"STATE_FOLLOWER\"}"); + context(pd, "/-/state", 200, "{\"status\":0,\"text\":\"ONLINE\"}"); + delayedContext(pd, "/-/lifecycle", 200, "{\"status\":0,\"text\":\"STARTED\"}", 1000L); + context(pd, "/v1/prom/targets/store", 200, targets("127.0.0.1:" + pd.getAddress().getPort())); + storeMetricContexts(pd); + String base = "http://127.0.0.1:" + pd.getAddress().getPort(); + LiveOperationsCollector collector = new LiveOperationsCollector( + true, base, "hubble", "secret", "store-hubble", "store-secret", "server-under-test", + new OperationsHttpClient(1000, 2000, 8192), new OperationsPayloadParser(new ObjectMapper()), CLOCK, + 1, 500, Set.of(base)); + try { + long started = System.nanoTime(); + OperationsModels.Node node = store(collector.collect(serverClient(), true)); + long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started); + Assert.assertTrue("Lifecycle must share the bounded Store collection budget", elapsed < 900L); + Assert.assertEquals("READY", node.getReadiness().getStatus()); + Assert.assertEquals("AVAILABLE", node.getMetricStatuses().get("system").getAvailability()); + Assert.assertEquals("UNKNOWN", node.getProcessLifecycle().getState()); + Assert.assertEquals("upstream_deadline", node.getProcessLifecycle().getReason()); + } finally { + collector.close(); + pd.stop(0); + upstream.shutdownNow(); + } + } + @Test public void testMetricDeadlineRetainsAlreadyCompletedStoreReadiness() throws Exception { java.util.concurrent.ExecutorService upstream = java.util.concurrent.Executors.newCachedThreadPool(); diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/OperationsPayloadParserTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/OperationsPayloadParserTest.java index c0c17f0ff..3906f16f5 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/OperationsPayloadParserTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/op/OperationsPayloadParserTest.java @@ -81,6 +81,31 @@ public void testReadinessRequiresExplicitKnownBodyAndPreservesSafeReasons() thro "{\"status\":1,\"text\":\"ONLINE\"}", 200, 1L, "/probe-prefix/v1/ready").getStatus()); } + @Test + public void testStoreLifecycleAcceptsOnlyExplicitLifecycleStates() throws Exception { + OperationsPayloadParser parser = new OperationsPayloadParser(MAPPER); + for (String state : new String[]{"STARTING", "STARTED", "STOPPING", "UNAVAILABLE"}) { + OperationsModels.Lifecycle result = parser.parseStoreLifecycle( + "{\"status\":0,\"text\":\"" + state + "\"}", 200, 123L); + Assert.assertEquals(state, result.getState()); + Assert.assertNull(result.getReason()); + Assert.assertEquals(Long.valueOf(123L), result.getObservedAt()); + Assert.assertFalse(result.isStale()); + } + for (String body : new String[]{"null", "[]", "{}", "{\"status\":0,\"text\":\"ONLINE\"}", + "{\"status\":1,\"text\":\"STARTED\"}", + "{\"status\":4294967296,\"text\":\"STARTED\"}", + "{\"status\":18446744073709551616,\"text\":\"STARTED\"}", + "{\"status\":0,\"text\":\"private-secret\"}"}) { + OperationsModels.Lifecycle result = parser.parseStoreLifecycle(body, 200, 123L); + Assert.assertEquals("UNKNOWN", result.getState()); + Assert.assertEquals("response_unrecognized", result.getReason()); + Assert.assertFalse(MAPPER.writeValueAsString(result).contains("private-secret")); + } + Assert.assertEquals("UNKNOWN", parser.parseStoreLifecycle( + "{\"status\":0,\"text\":\"STARTED\"}", 503, 123L).getState()); + } + @Test public void testParsesPdWrapperAndRedactsInfrastructureFields() throws Exception { diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.js b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.js index 2d554f8db..0908b8760 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.js +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.js @@ -35,6 +35,7 @@ const REQUIRED_CHECKS = Object.freeze([ 'PD unmapped metrics have localized three-node source diagnostics', 'mobile operations preserves readable main tiers and Store-row compute access', 'anonymous GraphSpaces visible with account and mutation actions hidden', + 'algorithm search waits for selected graph in authenticated and anonymous modes', ]); function validateReport(report) { diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.test.js b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.test.js index 8b8bf86a9..a3663e3bd 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.test.js +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions.test.js @@ -21,7 +21,7 @@ const os = require('node:os'); const path = require('node:path'); const {spawnSync} = require('node:child_process'); const {REQUIRED_CHECKS, validateReport, main} = require('./run_ui_deployment_regressions'); -const {sessionFixture, queryFixture} = require('./run_ui_deployment_regressions_impl'); +const {sessionFixture, queryFixture, graphViewFixture, graphsBySpace} = require('./run_ui_deployment_regressions_impl'); function passing() { return {status: 'passed', checks: REQUIRED_CHECKS.map(name => ({name, status: 'passed'})), @@ -183,3 +183,21 @@ test('synchronous query fixture rejects a wrong method, scope or query instead o assert.equal(response.data, undefined); } }); + +test('graph card fixtures accept only existing graphs in their own GraphSpace', () => { + for (const [space, graphs] of Object.entries(graphsBySpace)) { + for (const graph of graphs) assert.equal(graph.graphspace, space); + } + for (const scope of ['DEFAULT/graphs/hugegraph', 'DEFAULT/graphs/default_only', + 'soc_graph/graphs/hugegraph', 'soc_graph/graphs/second']) { + const base = `/graphspaces/${scope}`; + assert.equal(graphViewFixture('GET', `${base}/schema/graphview`).status, 200); + assert.equal(queryFixture('GET', `${base}/gremlin-query`).status, 200); + assert.equal(graphViewFixture('POST', `${base}/schema/graphview`).status, 405); + } + for (const path of ['/graphspaces/DEFAULT/graphs/second', + '/graphspaces/soc_graph/graphs/default_only', '/graphspaces/missing/graphs/hugegraph']) { + assert.equal(graphViewFixture('GET', `${path}/schema/graphview`).status, 404); + assert.equal(queryFixture('GET', `${path}/gremlin-query`).status, 404); + } +}); diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions_impl.js b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions_impl.js index f3d3b8182..b0ff0f83b 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions_impl.js +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_deployment_regressions_impl.js @@ -33,15 +33,26 @@ const spaces = [ { name: 'DEFAULT', nickname: '默认图空间', create_time: '2026-10-13', auth: false, max_graph_number: 100, cpu_limit: 64, memory_limit: 128, compute_cpu_limit: 64, compute_memory_limit: 128, storage_limit: 100, storage_used_kb: 10752 }, { name: 'soc_graph', nickname: '记忆库', create_time: '2026-10-15', auth: false, max_graph_number: 100, cpu_limit: 64, memory_limit: 128, compute_cpu_limit: 64, compute_memory_limit: 128, storage_limit: 10, storage_used_kb: 252590 }, ]; -const graph = (name, nickname) => ({ name, nickname, status: 'CREATED' }); +const graph = (graphspace, name, nickname) => ({ graphspace, name, nickname, status: 'CREATED' }); // The destination graph must not exist in DEFAULT: stale selector data cannot pass. const graphsBySpace = { - DEFAULT: [graph('hugegraph', '记忆图'), graph('default_only', '默认空间专属图')], - soc_graph: [graph('hugegraph', '记忆图'), graph('second', '测试图')], + DEFAULT: [graph('DEFAULT', 'hugegraph', '记忆图'), graph('DEFAULT', 'default_only', '默认空间专属图')], + soc_graph: [graph('soc_graph', 'hugegraph', '记忆图'), graph('soc_graph', 'second', '测试图')], }; +const hasFixtureScope = scope => Object.entries(graphsBySpace).some(([space, graphs]) => ( + graphs.some(item => scope === `${space}/graphs/${item.name}`) +)); +function graphViewFixture(method, pathname) { + const prefix = '/graphspaces/', suffix = '/schema/graphview'; + const scope = pathname.startsWith(prefix) && pathname.endsWith(suffix) + ? pathname.slice(prefix.length, -suffix.length) : ''; + if (!hasFixtureScope(scope)) return {status: 404}; + if (method !== 'GET') return {status: 405}; + return {status: 200, data: {vertices: [], edges: []}}; +} const now = 1792757451000; const node = (n) => ({ id: `pd-${n}`, name: `PD ${n}`, type: 'PD', role: n === 1 ? 'LEADER' : 'FOLLOWER', status: 'UP', observed_at: now, metrics: {}, metric_statuses: { system: { availability: 'UNSUPPORTED', reason: 'metrics_not_collected', observed_at: now, fresh: false, stale: false } } }); -const source = { status: 'UP', availability: 'PARTIAL', reason: 'pd_metrics_unmapped', observed_at: now, last_success_at: now, fresh: false, stale: false, diagnostics: { affected_node_count: 1, failed_metric_groups: ['system'], pd_metric_mapping: { configured_origin: 'http://pd-service:8620', status: 'no_match', matched_nodes: 0, discovered_nodes: 3, omitted_members: 0, members: [1, 2, 3].map(n => ({ node_id: 'pd-' + n, origin: 'http://pd-' + n + ':8620', endpoint_valid: true, origin_matches: false, base_path_matches: true })) } }, cluster_state: 'Cluster_Not_Ready', data_state: 'PState_Normal' }; +const source = { status: 'UP', availability: 'PARTIAL', reason: 'pd_metrics_unmapped', observed_at: now, last_success_at: now, fresh: false, stale: false, diagnostics: { affected_node_count: 1, failed_metric_groups: ['system'], pd_endpoint_metrics: { metrics: { process_cpu_usage: 0.125, heap_used_bytes: 1048576 }, metric_status: { availability: 'AVAILABLE', observed_at: now, fresh: true, stale: false } }, pd_metric_mapping: { configured_origin: 'http://pd-service:8620', status: 'no_match', matched_nodes: 0, discovered_nodes: 3, omitted_members: 0, members: [1, 2, 3].map(n => ({ node_id: 'pd-' + n, origin: 'http://pd-' + n + ':8620', endpoint_valid: true, origin_matches: false, base_path_matches: true })) } }, cluster_state: 'Cluster_Not_Ready', data_state: 'PState_Normal' }; function mobileOverviewFixture() { const observed = {availability: 'AVAILABLE', observed_at: now, last_success_at: now, fresh: true, stale: false}; const mainNode = (type, index) => ({id: `${type.toLowerCase()}-mobile-${index}`, type, @@ -50,12 +61,14 @@ function mobileOverviewFixture() { status: type === 'SERVER' ? 'UP' : 'UNKNOWN', observed_at: now, metrics: type === 'STORE' ? {backend: {leaders: index}} : {}, metric_statuses: {system: type === 'SERVER' ? observed - : {...observed, availability: 'UNAVAILABLE', reason: 'metric_unavailable'}}}); + : {...observed, availability: 'UNAVAILABLE', reason: 'metric_unavailable'}}, + ...(type === 'STORE' ? {readiness: {status: 'NOT_READY', state: 'STARTING', stale: false}, + process_lifecycle: {state: 'STARTED', observed_at: now, stale: false}} : {})}); const computeNode = role => ({id: role.toLowerCase(), name: `${role.toLowerCase()}-mobile`, role, status: 'UP', ready: true, group: 'default', version: '1.8.0', observed_at: now, checks: {registered: true, connection_ready: true, load_stream_ready: true, compute_stream_ready: true, superstep_stream_ready: true}, - metrics: {cpu_cores: role === 'MASTER' ? 0.12 : 0.42, rss_bytes: 1288490189}, + metrics: {cpu_cores: role === 'MASTER' ? 0.12000000000000001 : 0.42, rss_bytes: 1288490189}, metric_status: observed}); return {status: 'DEGRADED', main_cluster_status: 'DEGRADED', reason: 'cluster_not_ready', observed_at: now, stale: false, @@ -81,7 +94,7 @@ function queryFixture(method, pathname, body, truncated = false) { const scope = pathname.startsWith(prefix) && pathname.endsWith(suffix) ? pathname.slice(prefix.length, -suffix.length) : ''; const queries = {'DEFAULT/graphs/hugegraph': 'g.V()', 'soc_graph/graphs/second': 'g.V().limit(2)'}; - if (!Object.hasOwn(queries, scope) && scope !== 'soc_graph/graphs/hugegraph') return {status: 404}; + if (!hasFixtureScope(scope)) return {status: 404}; if (method === 'GET') return {status: 200, data: {vertexcount: null, edgecount: null}}; if (method !== 'POST') return {status: 405}; if (!queries[scope] || body?.content !== queries[scope] || body?.preview !== true) return {status: 400}; @@ -96,7 +109,7 @@ function queryFixture(method, pathname, body, truncated = false) { async function setup(browser, auth) { const context = await browser.newContext({ viewport: { width: 1440, height: 1000 }, timezoneId: 'Asia/Taipei' }); await context.addInitScript(() => localStorage.setItem('languageType', 'zh-CN')); - const state = { auth, loggedIn: false, loginRequests: [], truncation: false, pendingDetail: null, taskLookups: 0, taskSubmitted: false, + const state = { auth, loggedIn: false, loginRequests: [], truncation: false, pendingDetail: null, graphGate: null, taskLookups: 0, taskSubmitted: false, operationsOverview: null, extraCapabilities: [] }; const capabilities = ['graphspaces_read', 'operations_health_read', 'operations_topology_read', ...(auth ? ['accounts_manage', 'model_settings_manage'] : [])]; await context.route('**/api/v*/**', async (route) => { @@ -181,6 +194,9 @@ async function setup(browser, auth) { else if (p.endsWith('/graphs')) { const graphSpace = p.split('/')[2]; assert(graphsBySpace[graphSpace], `Unexpected graph-list scope: ${graphSpace}`); + if (state.graphGate) { + await state.graphGate; + } data = pageResult(graphsBySpace[graphSpace], url.searchParams.get('selected')); } else if (p.endsWith('/graph_read_mode')) @@ -191,6 +207,12 @@ async function setup(browser, auth) { data = { configured: false, available: false }; else if (p.endsWith('/text2gql/config')) data = { available: false, custom_available: true, reason: 'not_configured' }; + else if (p.endsWith('/schema/graphview')) { + const response = graphViewFixture(method, p); + if (response.status !== 200) unexpected.add(`${method} ${p}`); + await route.fulfill({status: response.status, contentType: 'application/json', body: JSON.stringify(response)}); + return; + } else if (p.endsWith('/schema/edgelabels') || p.endsWith('/schema/propertykeys')) data = { records: [] }; else if (p.endsWith('/schema/vertexlabels')) @@ -454,6 +476,9 @@ async function main() { await page.goto(base + '/operations/nodes?type=PD&source=pd#operations-source-diagnostics'); await page.locator('summary').filter({hasText: '检查地址与修复方法'}).click(); await page.getByText('最近拓扑采集成功', { exact: false }).waitFor(); + await page.getByText('配置的 PD 端点样本', {exact: true}).waitFor(); + await page.getByText(/不代表某个节点或整个集群的总量/).waitFor(); + await page.getByText('12.5%', {exact: true}).waitFor(); const repair = await page.locator('details.operations-diagnostic-details').filter({ has: page.locator('summary').filter({hasText: '检查地址与修复方法'}), }).innerText(); @@ -461,10 +486,10 @@ async function main() { assert(repair.includes(detail), `Missing actionable PD repair detail: ${detail}`); } assert(/重启[^。]*Hubble|Hubble[^。]*重启/.test(repair), 'Missing Hubble restart guidance'); - await page.getByText('尚未采集系统指标', { exact: false }).first().waitFor(); + await page.getByText('尚未采集节点系统指标', { exact: false }).first().waitFor(); assert.equal(await page.getByText(/来源可能在发现节点前/).count(), 0); await page.getByText('http://pd-service:8620', { exact: true }).waitFor(); - await page.getByText(/受影响的 PD 采集端点:\s*1/).waitFor(); + await page.getByText(/未归属节点的 PD 采集端点:\s*1/).waitFor(); await page.getByText(/Hubble 只采集一个匹配的 PD 成员的系统指标/).waitFor(); await page.getByText('没有匹配节点', { exact: true }).waitFor(); await page.getByText('0 / 3', { exact: true }).waitFor(); @@ -516,11 +541,22 @@ async function main() { await page.getByRole('heading', {name: state.operationsOverview.nodes[0].name, exact: true}).waitFor(); await page.locator('.operations-node-detail .operations-page-header button').first().click(); await topology.waitFor(); + if (width === 375) { + const store = topology.locator('.tier-store .operations-topology-node').first(); + await store.click(); + const process = page.getByRole('region', {name: 'Store 进程生命周期'}); + await process.getByText('已启动', {exact: true}).waitFor(); + await process.getByText(/不能证明 PD 成员状态、分区健康或读写已就绪/).waitFor(); + await page.getByRole('region', {name: '服务就绪状态'}).getByText('未就绪', {exact: true}).waitFor(); + await page.locator('.operations-node-detail .operations-page-header button').first().click(); + await topology.waitFor(); + } const summary = page.getByRole('button', {name: '查看计算集群', exact: true}); await assertUsableControl(summary); await summary.click(); await page.locator('.operations-vermeer-workbench').waitFor(); assert.equal(await page.locator('.operations-vermeer-node').count(), 2); + assert.equal(await page.getByText('已采集 2 / 2 个节点', {exact: true}).count(), 2); assert.deepEqual(await page.locator('.operations-vermeer-resource-fact strong').allTextContents(), ['0.54 核', '2.4 GiB']); await assertNoHorizontalOverflow(page); @@ -553,6 +589,26 @@ async function main() { await anonymous.page.getByText('图模式', { exact: true }).click(); await screenshot(anonymous.page, 'graphspace-anonymous'); }); + await check('algorithm search waits for selected graph in authenticated and anonymous modes', async () => { + for (const item of [{page, state}, anonymous]) { + let release; + item.state.graphGate = new Promise(resolve => { release = resolve; }); + try { + await item.page.goto(base + '/algorithms/DEFAULT/hugegraph'); + const search = item.page.getByPlaceholder('算法查询', {exact: true}); + await search.waitFor({state: 'visible'}); + assert(await search.isDisabled(), 'Search must be disabled while graph options are pending'); + release(); + const unrelated = item.page.getByText('查找所有路径（GET，基础版）', {exact: true}).first(); + await unrelated.waitFor({state: 'visible'}); + await search.fill('查找最短路径'); + await item.page.getByText('查找最短路径', {exact: true}).first().waitFor({state: 'visible'}); + await unrelated.waitFor({state: 'hidden'}); + assert.equal(await search.inputValue(), '查找最短路径'); + } + finally { release(); item.state.graphGate = null; } + } + }); } catch (e) { failure = e.message; @@ -574,4 +630,4 @@ async function main() { if (require.main === module) { main().catch(error => { console.error(error.stack); process.exitCode = 1; }); } -module.exports = {sessionFixture, queryFixture}; +module.exports = {sessionFixture, queryFixture, graphViewFixture, graphsBySpace}; diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_operations_diagnostics.js b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_operations_diagnostics.js index 17ce67b5e..c1cde698c 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_operations_diagnostics.js +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_ui_operations_diagnostics.js @@ -193,8 +193,8 @@ async function check(name, action) { await capture(page, '04-pd-diagnostics-collapsed'); await disclosure.click(); await page.getByText('http://pd-service:8620', {exact: true}).waitFor(); - await page.getByText('尚未采集系统指标', {exact: false}).first().waitFor(); - await page.getByText(/受影响的 PD 采集端点: 1/).waitFor(); + await page.getByText('尚未采集节点系统指标', {exact: false}).first().waitFor(); + await page.getByText(/未归属节点的 PD 采集端点: 1/).waitFor(); await page.getByText(/Hubble 只采集一个匹配的 PD 成员/).waitFor(); for (const member of pds) { assert.equal(await page.getByText(`${member.name} · system`, {exact: true}).count(), 0); diff --git a/hugegraph-hubble/hubble-fe/src/i18n/resources/en-US/modules/pages.json b/hugegraph-hubble/hubble-fe/src/i18n/resources/en-US/modules/pages.json index 5048d441a..10f71a1ae 100644 --- a/hugegraph-hubble/hubble-fe/src/i18n/resources/en-US/modules/pages.json +++ b/hugegraph-hubble/hubble-fe/src/i18n/resources/en-US/modules/pages.json @@ -1352,16 +1352,28 @@ "diagnostic_large_response_action": "The response was stopped at the configured collection limit. Check the failed endpoint and follow the steps below.", "diagnostic_check_source_action": "Check the affected request, node metric groups and upstream logs, then refresh to confirm recovery.", "reason_pd_metrics_unmapped": "The configured PD endpoint could not be matched to a discovered PD node", - "diagnostic_pd_mapping_action": "PD topology was discovered, but system metrics were not collected because the configured endpoint could not be uniquely matched to a member's restUrl in the PD member list.", + "diagnostic_pd_mapping_action": "PD topology was discovered, but system metrics were not assigned to a node because the configured endpoint could not be uniquely matched to a member's restUrl in the PD member list.", + "lifecycle_title": "Store process lifecycle", + "lifecycle_independent": "Process lifecycle reports startup and shutdown events only. It does not establish PD membership, partition health or read/write readiness", + "lifecycle_starting": "Starting", + "lifecycle_started": "Started", + "lifecycle_stopping": "Stopping", + "lifecycle_unavailable": "Process unavailable", + "lifecycle_unknown": "Unknown", + "lifecycle_stale": "Stale", + "lifecycle_unsupported": "Not supported by this Store version", + "diagnostic_pd_endpoint_sample": "Configured PD endpoint sample", + "diagnostic_pd_endpoint_sample_help": "One response from the configured endpoint. A Service or proxy may route each request to a different member; these values do not identify a node or represent a cluster total", "diagnostic_step_pd_endpoint": "A Service, proxy or load-balancer address can reach PD without identifying a single member. Compare the configured address with the member REST addresses below; an address mismatch alone does not mean PD is down.", "diagnostic_step_pd_config": "To collect CPU and JVM metrics, set pd.server in hugegraph-hubble.properties to a reachable member REST address, or correct the advertised restUrl. A scheme-less pd.server uses server.protocol. This changes the PD connection endpoint, so retain the intended deployment routing.", "diagnostic_step_pd_restart": "Restart Hubble after a configuration change, then refresh. Hubble collects system metrics from only the one matched PD member; other members remain intentionally uncollected.", "diagnostic_step_response_limit": "operations.max_response_bytes in hugegraph-hubble.properties limits each upstream response; the current limit is shown with the failed request. “At least” is a lower bound from a stopped read, not the full response size.", "diagnostic_step_raft_summary": "The older /metrics/raft endpoint returns full registries and timer samples for every Raft group. Upgrading Store to a version that provides /metrics/raft/summary lets Hubble collect just the two counters.", "diagnostic_step_response_config": "Inspect the full response size from the same endpoint. If the payload is expected, increase operations.max_response_bytes to fit the measured response within the available memory budget, restart Hubble, then refresh. A larger limit applies to every collection request.", - "diagnostic_pd_affected_endpoint": "Affected PD collection endpoints", + "diagnostic_pd_affected_endpoint": "PD endpoints without node attribution", + "diagnostic_unattributed_groups": "Unattributed metric groups", "diagnostic_topology_last_success": "Last topology success", - "diagnostic_metrics_not_collected": "System metrics not collected", + "diagnostic_metrics_not_collected": "Node system metrics not collected", "diagnostic_pd_address_comparison": "PD address comparison", "diagnostic_pd_online_unmapped": "PD nodes are online; system metrics could not be attributed", "diagnostic_pd_unmapped": "PD system metrics could not be attributed to a node", diff --git a/hugegraph-hubble/hubble-fe/src/i18n/resources/zh-CN/modules/pages.json b/hugegraph-hubble/hubble-fe/src/i18n/resources/zh-CN/modules/pages.json index c827d8e6a..708a33545 100644 --- a/hugegraph-hubble/hubble-fe/src/i18n/resources/zh-CN/modules/pages.json +++ b/hugegraph-hubble/hubble-fe/src/i18n/resources/zh-CN/modules/pages.json @@ -1352,16 +1352,28 @@ "diagnostic_large_response_action": "响应读取在配置的采集上限处停止。请确认失败接口，再按以下步骤处理。", "diagnostic_check_source_action": "请检查失败请求、节点指标组及上游日志，刷新后确认是否恢复。", "reason_pd_metrics_unmapped": "配置的 PD 地址无法匹配到已发现的 PD 节点", - "diagnostic_pd_mapping_action": "已发现 PD 拓扑，但配置地址无法唯一匹配 PD 成员列表中某个成员的 restUrl，因此未采集系统指标。", + "diagnostic_pd_mapping_action": "已发现 PD 拓扑，但配置地址无法唯一匹配 PD 成员列表中某个成员的 restUrl，系统指标未归属于任何节点。", + "lifecycle_title": "Store 进程生命周期", + "lifecycle_independent": "进程生命周期仅反映启动和停止事件，不能证明 PD 成员状态、分区健康或读写已就绪", + "lifecycle_starting": "启动中", + "lifecycle_started": "已启动", + "lifecycle_stopping": "停止中", + "lifecycle_unavailable": "进程不可用", + "lifecycle_unknown": "未知", + "lifecycle_stale": "已过期", + "lifecycle_unsupported": "当前 Store 版本不支持", + "diagnostic_pd_endpoint_sample": "配置的 PD 端点样本", + "diagnostic_pd_endpoint_sample_help": "来自配置端点的一次响应。Service 或代理可能将每次请求转发到不同成员；这些数值不代表某个节点或整个集群的总量", "diagnostic_step_pd_endpoint": "Service、代理或负载均衡地址可以连接 PD，但无法标识单个成员。请对照下方的配置地址与成员 REST 地址；地址不匹配本身不代表 PD 宕机。", "diagnostic_step_pd_config": "如需采集 CPU、JVM 指标，在 hugegraph-hubble.properties 中将 pd.server 改为可访问的成员 REST 地址，或修正成员公布的 restUrl。未写协议时使用 server.protocol。这会改变 PD 连接入口，请保留部署所需的路由方式。", "diagnostic_step_pd_restart": "修改配置后重启 Hubble，再刷新此页。Hubble 只采集一个匹配的 PD 成员的系统指标，其他成员不会被主动采集。", "diagnostic_step_response_limit": "hugegraph-hubble.properties 中的 operations.max_response_bytes 限制每次上游响应，实际上限见失败请求的“响应大小上限”。“至少”表示停止读取时已收到的字节数，不是完整响应大小。", "diagnostic_step_raft_summary": "旧版 /metrics/raft 会返回每个 Raft 组的完整指标注册表和计时器样本。将 Store 升级至提供 /metrics/raft/summary 的版本后，Hubble 可仅采集所需的两个计数。", "diagnostic_step_response_config": "请先核对同一接口的完整响应大小。如数据量符合预期，可在内存预算内增大 operations.max_response_bytes 以容纳实测响应，重启 Hubble 后刷新。此上限对每次采集请求都生效。", - "diagnostic_pd_affected_endpoint": "受影响的 PD 采集端点", + "diagnostic_pd_affected_endpoint": "未归属节点的 PD 采集端点", + "diagnostic_unattributed_groups": "未归属指标组", "diagnostic_topology_last_success": "最近拓扑采集成功", - "diagnostic_metrics_not_collected": "尚未采集系统指标", + "diagnostic_metrics_not_collected": "尚未采集节点系统指标", "diagnostic_pd_address_comparison": "PD 地址核对", "diagnostic_pd_online_unmapped": "PD 节点在线，系统指标尚未匹配到节点", "diagnostic_pd_unmapped": "PD 系统指标尚未匹配到节点", diff --git a/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.js b/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.js index 1d65cc891..45b8aadc7 100644 --- a/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.js +++ b/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.js @@ -28,7 +28,7 @@ import {useTranslation} from 'react-i18next'; const AlgorithmSearch = props => { const {t} = useTranslation(); - const {onSearch} = props; + const {onSearch, disabled = false} = props; const debounceOnChange = useMemo( () => { @@ -45,6 +45,7 @@ const AlgorithmSearch = props => { diff --git a/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.test.js b/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.test.js new file mode 100644 index 000000000..3aa1e1fb5 --- /dev/null +++ b/hugegraph-hubble/hubble-fe/src/modules/algorithm/AlgorithmSearch/index.test.js @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import {act, fireEvent, render, screen} from '@testing-library/react'; +import AlgorithmSearch from './index'; + +jest.mock('react-i18next', () => ({useTranslation: () => ({t: key => key})})); + +test('blocks input until the scope is ready, then applies the search', () => { + jest.useFakeTimers(); + const search = jest.fn(); + const view = render(); + const input = screen.getByRole('textbox', {name: ''}); + expect(input).toBeDisabled(); + view.rerender(); + expect(input).toBeEnabled(); + fireEvent.change(input, {target: {value: 'Find Shortest Path'}}); + act(() => jest.advanceTimersByTime(200)); + expect(search).toHaveBeenCalledWith('Find Shortest Path'); + view.unmount(); + jest.useRealTimers(); +}); diff --git a/hugegraph-hubble/hubble-fe/src/modules/algorithm/algorithmsForm/Home/index.js b/hugegraph-hubble/hubble-fe/src/modules/algorithm/algorithmsForm/Home/index.js index 3dd1ee8b5..b98ace1be 100644 --- a/hugegraph-hubble/hubble-fe/src/modules/algorithm/algorithmsForm/Home/index.js +++ b/hugegraph-hubble/hubble-fe/src/modules/algorithm/algorithmsForm/Home/index.js @@ -68,7 +68,7 @@ const AlgorithmFormHome = props => { {key: 'olap', label: t('analysis.algorithm.olap_tab')}, ]} /> - + )); -jest.mock('../../AlgorithmSearch', () => () =>
algorithm search
); +jest.mock('../../AlgorithmSearch', () => ({disabled}) => ( + +)); jest.mock('react-router-dom', () => ({ Link: ({to, children}) => {children}, })); @@ -62,6 +64,23 @@ beforeEach(() => { mockFinishOltp = undefined; }); +test('enables catalog search only after both graph scope components resolve', () => { + const view = render(); + expect(screen.getByRole('textbox', {name: 'algorithm search'})).toBeDisabled(); + + view.rerender(); + expect(screen.getByRole('textbox', {name: 'algorithm search'})).toBeDisabled(); + + view.rerender(); + expect(screen.getByRole('textbox', {name: 'algorithm search'})).toBeEnabled(); + + view.rerender(); + expect(screen.getByRole('textbox', {name: 'algorithm search'})).toBeDisabled(); + + view.rerender(); + expect(screen.getByRole('textbox', {name: 'algorithm search'})).toBeEnabled(); +}); + const renderForm = graphNums => render( { role='status' aria-label={availabilityLabel} > - {status.fresh && ( + {status.fresh && !status.stale && ( { } else { setData(snapshot => snapshot && {...snapshot, stale: true, - node: markNodeReadinessStale(snapshot.node)}); + node: markNodeStale(snapshot.node), + sources: markSourcesStale(snapshot.sources)}); } setError(requestError); } @@ -539,7 +540,7 @@ const NodeDetail = () => { reason: source.reason, }; } - return {availability: node.metrics?.[group] ? 'AVAILABLE' : 'UNSUPPORTED'}; + return {availability: node.metrics?.[group] ? 'AVAILABLE' : 'UNSUPPORTED', stale: data?.stale === true}; }; return (
@@ -567,7 +568,7 @@ const NodeDetail = () => { } - +
{t('operations.last_observed')}: {observed} @@ -605,6 +606,7 @@ const NodeDetail = () => { /> + {node.type === 'STORE' && }
{ test('keeps old readiness stale through a retry until a successful node response arrives', async () => { const ready = {...response, stale: false, - node: {...response.node, readiness: {status: 'READY', stale: false, observed_at: 1000}}}; + node: {...response.node, readiness: {status: 'READY', stale: false, observed_at: 1000}, + process_lifecycle: {state: 'STARTED', stale: false, observed_at: 1000}}}; let finishRetry; getNode.mockResolvedValueOnce(ready).mockRejectedValueOnce(new Error('offline')) .mockReturnValueOnce(new Promise(resolve => {finishRetry = resolve;})); renderDetail(); const panel = await screen.findByRole('region', {name: 'Service readiness'}); + const lifecycle = screen.getByRole('region', {name: 'Store process lifecycle'}); expect(within(panel).getByText('Ready', {exact: true})).toBeVisible(); + expect(within(lifecycle).getByText('Started', {exact: true})).toBeVisible(); fireEvent.click(screen.getByRole('button', {name: /Refresh/})); await waitFor(() => expect(within(panel).getByText('Refresh needed')).toBeVisible()); + expect(within(lifecycle).getByText('Stale', {exact: true})).toBeVisible(); expect(within(panel).queryByText('Ready', {exact: true})).not.toBeInTheDocument(); fireEvent.click(screen.getByRole('button', {name: /Refresh/})); expect(within(panel).getByText('Refresh needed')).toBeVisible(); @@ -272,6 +276,106 @@ test('keeps old readiness stale through a retry until a successful node response expect(within(panel).getByText('Ready', {exact: true})).toBeVisible(); }); +test.each([ + ['explicit metric status', {system: {availability: 'AVAILABLE', fresh: true, stale: false}}, 'AVAILABLE', true], + ['partial source fallback', undefined, 'PARTIAL', true], + ['available source fallback', undefined, 'AVAILABLE', false], +])('keeps %s stale through refresh failure and pending retry', async (_, statuses, availability, initiallyFresh) => { + getNode.mockReset(); + const snapshot = {...response, stale: false, + node: {...response.node, id: 'pd-safe', name: 'PD A', type: 'PD', + metrics: {system: {uptime_seconds: 12}}, metric_statuses: statuses}, + sources: {pd: {status: 'UP', availability, fresh: true, stale: false, observed_at: 1000}}}; + let finishRetry; + getNode.mockResolvedValueOnce(snapshot).mockRejectedValueOnce(new Error('offline')) + .mockReturnValueOnce(new Promise(resolve => {finishRetry = resolve;})); + renderDetail(); + await screen.findByRole('heading', {name: 'PD A'}); + const system = screen.getByRole('heading', {name: 'System'}).closest('section'); + expect(Boolean(within(system).queryByRole('img', {name: 'Fresh'}))).toBe(initiallyFresh); + + fireEvent.click(screen.getByRole('button', {name: 'Refresh'})); + await waitFor(() => expect(within(system).getByText('Stale')).toBeVisible()); + expect(within(system).queryByRole('img', {name: 'Fresh'})).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', {name: 'Refresh'})); + expect(within(system).getByText('Stale')).toBeVisible(); + expect(within(system).queryByRole('img', {name: 'Fresh'})).not.toBeInTheDocument(); + + await act(async () => finishRetry(snapshot)); + expect(within(system).queryByText('Stale')).not.toBeInTheDocument(); + expect(Boolean(within(system).queryByRole('img', {name: 'Fresh'}))).toBe(initiallyFresh); +}); + +test('does not display a fresh icon when a metric response is also stale', async () => { + getNode.mockReset(); + getNode.mockResolvedValue({...response, node: {...response.node, + metric_statuses: {system: {availability: 'AVAILABLE', fresh: true, stale: true}}}}); + renderDetail(); + await screen.findByRole('heading', {name: 'Store A'}); + const system = screen.getByRole('heading', {name: 'System'}).closest('section'); + expect(within(system).getByText('Stale')).toBeVisible(); + expect(within(system).queryByRole('img', {name: 'Fresh'})).not.toBeInTheDocument(); +}); + +test('keeps a retained PD endpoint sample stale through a failed refresh and pending retry', async () => { + const pd = {...response, stale: false, node: {...response.node, + id: 'pd-safe', name: 'PD A', type: 'PD', metrics: {}, metric_statuses: {}}, + sources: {pd: {status: 'UP', availability: 'PARTIAL', reason: 'pd_metrics_unmapped', stale: false, + diagnostics: {pd_endpoint_metrics: {metrics: {process_cpu_usage: 0.125, heap_used_bytes: 1048576}, + metric_status: {availability: 'AVAILABLE', fresh: true, stale: false, observed_at: 1000}}}}}}; + let finishRetry; + getNode.mockResolvedValueOnce(pd).mockRejectedValueOnce(new Error('offline')) + .mockReturnValueOnce(new Promise(resolve => {finishRetry = resolve;})); + renderDetail(); + await screen.findByRole('heading', {name: 'PD A'}); + fireEvent.click(screen.getByText('Check addresses and how to fix')); + const sample = screen.getByText('Configured PD endpoint sample').parentElement; + expect(within(sample).getByText('12.5%')).toBeVisible(); + expect(within(sample).getByText('1 MB')).toBeVisible(); + + fireEvent.click(screen.getByRole('button', {name: /Refresh/})); + await waitFor(() => expect(within(sample).getAllByText('Stale').length).toBeGreaterThan(0)); + expect(within(sample).queryByText('12.5%')).not.toBeInTheDocument(); + expect(within(sample).queryByText('1 MB')).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', {name: /Refresh/})); + expect(within(sample).getAllByText('Stale').length).toBeGreaterThan(0); + expect(within(sample).queryByText('12.5%')).not.toBeInTheDocument(); + + await act(async () => finishRetry(pd)); + expect(within(sample).getByText('12.5%')).toBeVisible(); + expect(within(sample).getByText('1 MB')).toBeVisible(); + expect(within(sample).queryByText('Stale')).not.toBeInTheDocument(); +}); + +test('shows Store process lifecycle independently of readiness', async () => { + getNode.mockResolvedValue({...response, stale: false, node: {...response.node, + readiness: {status: 'NOT_READY', state: 'STARTING', stale: false}, + process_lifecycle: {state: 'STARTED', stale: false, observed_at: 1000}}}); + renderDetail(); + const lifecycle = await screen.findByRole('region', {name: 'Store process lifecycle'}); + expect(within(lifecycle).getByText('Started', {exact: true})).toBeVisible(); + expect(within(lifecycle).getByText(/does not establish PD membership, partition health or read\/write readiness/)) + .toBeVisible(); + expect(within(screen.getByRole('region', {name: 'Service readiness'})).getByText('Not ready', {exact: true})) + .toBeVisible(); +}); + +test('old Store lifecycle endpoint degrades safely', async () => { + getNode.mockResolvedValue({...response, stale: false, node: {...response.node, + process_lifecycle: {state: 'UNKNOWN', reason: 'unsupported_version', stale: false}}}); + renderDetail(); + const lifecycle = await screen.findByRole('region', {name: 'Store process lifecycle'}); + expect(within(lifecycle).getByText('Not supported by this Store version')).toBeVisible(); + expect(within(lifecycle).queryByText('Started')).not.toBeInTheDocument(); +}); + +test('older Hubble responses without lifecycle do not invent a state', async () => { + getNode.mockResolvedValue(response); + renderDetail(); + await screen.findByRole('heading', {name: 'Store A'}); + expect(screen.queryByRole('region', {name: 'Store process lifecycle'})).not.toBeInTheDocument(); +}); + test('recovers an initial node failure and keeps a canonical node-list action', async () => { getNode.mockRejectedValueOnce(new Error('down')).mockResolvedValueOnce(response); @@ -286,10 +390,10 @@ test('recovers an initial node failure and keeps a canonical node-list action', expect(getNode).toHaveBeenCalledTimes(2); }); -test('removes an earlier privileged node detail when refresh is forbidden', async () => { +test.each([401, 403])('removes an earlier privileged node detail when refresh returns %s', async status => { getNode .mockResolvedValueOnce(response) - .mockRejectedValueOnce(Object.assign(new Error('forbidden'), {status: 403})); + .mockRejectedValueOnce(Object.assign(new Error('forbidden'), {status})); renderDetail(); expect(await screen.findByRole('heading', {name: 'Store A'})).toBeInTheDocument(); diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.js index 3bef1c6e3..3a97b53d4 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.js @@ -32,9 +32,9 @@ import { RefreshButton, TierIcon, } from './components'; -import {formatObservedAt, hasStaleMetrics, metricIssueReason} from './topology'; +import {formatObservedAt, hasStaleMetrics, markSourcesStale, metricIssueReason} from './topology'; import {operationsReturnState} from './navigation'; -import {ReadinessStatus} from './Readiness'; +import {markNodeStale, ReadinessStatus} from './Readiness'; import './operations.scss'; const stopRowNavigation = event => event.stopPropagation(); @@ -159,6 +159,12 @@ const Nodes = () => { if ([401, 403].includes(requestError?.status)) { setData({items: [], total: 0, observed_at: null, stale: false}); } + else { + setData(snapshot => ({...snapshot, stale: true, + items: snapshot.items.map(markNodeStale), + diagnostic_nodes: snapshot.diagnostic_nodes?.map(markNodeStale), + sources: markSourcesStale(snapshot.sources)})); + } setError(requestError); } finally { @@ -260,7 +266,7 @@ const Nodes = () => { render: (value, record) => ( + stale={data.stale || hasStaleMetrics(record)} /> )}, diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.test.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.test.js index c28fa45a8..a316a096c 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.test.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/Nodes.test.js @@ -231,7 +231,7 @@ test('shows stale observation metadata and exposes real detail links', async () ); const table = within(screen.getByRole('table')); const refresh = screen.getByLabelText('Refresh'); - expect(screen.getByText(/Stale/i)).toBeInTheDocument(); + expect(screen.getAllByText(/Stale/i).length).toBeGreaterThan(0); expect(screen.getAllByText(/Observed/i).length).toBeGreaterThan(0); const type = screen.getByLabelText(/node type/i); expect(type).toHaveRole('combobox'); @@ -348,15 +348,18 @@ test('renders a bounded empty table for a valid zero-node response', async () => .not.toBeInTheDocument(); }); -test('clears an earlier privileged snapshot when a refresh is forbidden', async () => { +test.each([401, 403])('clears an earlier privileged snapshot when refresh returns %s', async status => { getNodes .mockResolvedValueOnce({ items: [{id: 'admin-node', name: 'Admin node', type: 'SERVER', status: 'UP'}], total: 1, observed_at: 1000, stale: false, + sources: {pd: {status: 'UP', availability: 'PARTIAL', reason: 'pd_metrics_unmapped', + diagnostics: {pd_endpoint_metrics: {metrics: {process_cpu_usage: 0.125}, + metric_status: {availability: 'AVAILABLE', fresh: true}}}}}, }) - .mockRejectedValueOnce(Object.assign(new Error('forbidden'), {status: 403})); + .mockRejectedValueOnce(Object.assign(new Error('forbidden'), {status})); render( @@ -364,6 +367,7 @@ test('clears an earlier privileged snapshot when a refresh is forbidden', async ); expect(await screen.findByRole('link', {name: /Admin node/})).toBeInTheDocument(); + expect(screen.getByText('Configured PD endpoint sample')).toBeInTheDocument(); await waitFor(() => expect(screen.getByRole('button', {name: /Refresh/})) .not.toHaveClass('ant-btn-loading')); @@ -371,9 +375,49 @@ test('clears an earlier privileged snapshot when a refresh is forbidden', async await waitFor(() => expect(screen.getByRole('alert')).toBeInTheDocument()); expect(screen.queryByRole('link', {name: /Admin node/})).not.toBeInTheDocument(); + expect(screen.queryByText('Configured PD endpoint sample')).not.toBeInTheDocument(); expect(screen.getByText('0 nodes')).toBeInTheDocument(); }); +test('marks retained PD samples and node readiness stale until a retry succeeds', async () => { + i18n.changeLanguage('en-US'); + const pdNode = {id: 'pd-safe', name: 'PD A', type: 'PD', status: 'UP', + readiness: {status: 'READY', stale: false}, + metric_statuses: {system: {availability: 'AVAILABLE', stale: false}}}; + const snapshot = {items: [pdNode], diagnostic_nodes: [pdNode], total: 1, observed_at: 1000, stale: false, + sources: {pd: {status: 'UP', availability: 'PARTIAL', reason: 'pd_metrics_unmapped', stale: false, + diagnostics: {pd_endpoint_metrics: {metrics: {process_cpu_usage: 0.125, heap_used_bytes: 1048576}, + metric_status: {availability: 'AVAILABLE', fresh: true, stale: false, observed_at: 1000}}}}}}; + let finishRetry; + getNodes.mockResolvedValueOnce(snapshot).mockRejectedValueOnce(new Error('offline')) + .mockReturnValueOnce(new Promise(resolve => {finishRetry = resolve;})); + render(); + await screen.findByRole('link', {name: /PD A/}); + fireEvent.click(screen.getByText('Check addresses and how to fix')); + const sample = screen.getByText('Configured PD endpoint sample').parentElement; + const row = screen.getByRole('link', {name: /PD A/}).closest('tr'); + expect(within(sample).getByText('12.5%')).toBeVisible(); + expect(within(sample).getByText('1 MB')).toBeVisible(); + expect(within(row).getByLabelText('Service: Ready')).toBeVisible(); + + fireEvent.click(screen.getByRole('button', {name: 'Refresh'})); + await waitFor(() => expect(within(sample).getAllByText('Stale').length).toBeGreaterThan(0)); + expect(within(sample).queryByText('12.5%')).not.toBeInTheDocument(); + expect(within(sample).queryByText('1 MB')).not.toBeInTheDocument(); + expect(within(row).getByLabelText('Service: Refresh needed')).toBeVisible(); + expect(within(row).getByText('Stale')).toBeVisible(); + fireEvent.click(screen.getByRole('button', {name: 'Refresh'})); + expect(within(sample).getAllByText('Stale').length).toBeGreaterThan(0); + expect(within(sample).queryByText('12.5%')).not.toBeInTheDocument(); + expect(within(row).getByLabelText('Service: Refresh needed')).toBeVisible(); + + await act(async () => finishRetry(snapshot)); + expect(within(sample).getByText('12.5%')).toBeVisible(); + expect(within(sample).getByText('1 MB')).toBeVisible(); + expect(within(row).getByLabelText('Service: Ready')).toBeVisible(); + expect(within(row).queryByText('Stale')).not.toBeInTheDocument(); +}); + test('does not restore an older node list after a newer request is forbidden', async () => { let resolveOlder; getNodes diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/Readiness.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/Readiness.js index 586c90753..652b2aff2 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/Readiness.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/Readiness.js @@ -24,8 +24,37 @@ const readinessKey = readiness => readiness.stale ? 'stale' : readiness.reason === 'readiness_not_probed' ? 'not_probed' : readiness.status?.toLowerCase() ?? 'unknown'; -const markNodeReadinessStale = node => node?.readiness - ? {...node, readiness: {...node.readiness, stale: true}} : node; +const markNodeReadinessStale = node => node ? { + ...node, + ...(node.readiness ? {readiness: {...node.readiness, stale: true}} : {}), + ...(node.process_lifecycle ? {process_lifecycle: {...node.process_lifecycle, stale: true}} : {}), +} : node; + +const markNodeStale = node => node ? { + ...markNodeReadinessStale(node), + ...(node.metric_statuses ? {metric_statuses: Object.fromEntries( + Object.entries(node.metric_statuses).map(([name, status]) => [name, + status && {...status, fresh: false, stale: true}]) + )} : {}), +} : node; + +const ProcessLifecyclePanel = ({lifecycle}) => { + const {t, i18n} = useTranslation(); + if (!lifecycle) return null; + const state = lifecycle.stale ? 'STALE' : lifecycle.state; + const label = !lifecycle.stale && lifecycle.reason === 'unsupported_version' ? t('operations.lifecycle_unsupported') + : t(`operations.lifecycle_${state?.toLowerCase() ?? 'unknown'}`, + {defaultValue: t('operations.lifecycle_unknown')}); + return
+

{t('operations.lifecycle_title')}

{label}
+

{t('operations.lifecycle_independent')}

+ {lifecycle.observed_at && } +
; +}; const readinessLabel = (readiness, t) => readiness ? t( `operations.readiness_${readinessKey(readiness)}`, @@ -89,4 +118,4 @@ const ReadinessPanel = ({readiness}) => { ); }; -export {ReadinessStatus, ReadinessPanel, readinessLabel, markNodeReadinessStale}; +export {ReadinessStatus, ReadinessPanel, ProcessLifecyclePanel, readinessLabel, markNodeReadinessStale, markNodeStale}; diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.js index 1ac0bd24b..f31b108ed 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.js @@ -22,6 +22,7 @@ import {useTranslation} from 'react-i18next'; import {displayNodeType, formatReason, HealthStatus} from './components'; import {formatBytes, formatObservedAt, selectAttentionSources} from './topology'; import {sourceHealthStatus} from './sourceHealth'; +import {metricNumber} from './vermeer'; const SOURCE_TYPES = {server: 'SERVER', pd: 'PD', stores: 'STORE'}; const DIAGNOSTICS_ID = 'operations-source-diagnostics'; @@ -100,6 +101,45 @@ const PdMetricMapping = ({mapping, nodes}) => { ); }; +// These are one configured endpoint's response, not a discovered PD node +// and not an aggregation: a Service may route consecutive scrapes elsewhere. +const PdEndpointMetrics = ({sample, sourceStale}) => { + const {t, i18n} = useTranslation(); + if (!sample) return null; + const status = sample.metric_status ?? {}; + const fresh = !sourceStale && !status.stale && status.fresh === true + && status.availability === 'AVAILABLE'; + const metrics = fresh ? sample.metrics ?? {} : {}; + const format = key => { + const value = metricNumber(metrics[key]); + if (value === null) return t('operations.unavailable'); + if (key.endsWith('_bytes')) return formatBytes(value) ?? t('operations.unavailable'); + if (key.endsWith('_cpu_usage')) { + return new Intl.NumberFormat(i18n.language, {style: 'percent', maximumFractionDigits: 2}).format(value); + } + return new Intl.NumberFormat(i18n.language, {maximumFractionDigits: 2}).format(value); + }; + const keys = ['process_cpu_usage', 'system_cpu_usage', 'cpu_count', 'heap_used_bytes', + 'nonheap_used_bytes', 'threads_live', 'uptime_seconds']; + return
+

{t('operations.diagnostic_pd_endpoint_sample')}

+

{t('operations.diagnostic_pd_endpoint_sample_help')}

+ metrics[key] !== undefined && metrics[key] !== null) + .map(key => ({key, label: t(`operations.metric_labels.${key}`), children: format(key)})), + ]} /> + {!fresh &&

{sourceStale || status.stale ? t('operations.stale') + : formatReason(status.reason, t) || t('operations.unavailable')}

} + +
; +}; + const RecoverySteps = ({unmappedPd, largeResponse, raftTooLarge}) => { const {t} = useTranslation(); if (!unmappedPd && !largeResponse) { @@ -201,6 +241,8 @@ const SourceDiagnostics = ({sources = {}, sourceName, nodes = [], focus = false, {!recovered && } {source.diagnostics !== requestDetails && } + {type === 'PD' && } {unmappedPd && } {source.cluster_state === 'Cluster_Not_Ready' &&

{t('operations.reason_cluster_not_ready')} @@ -208,7 +250,8 @@ const SourceDiagnostics = ({sources = {}, sourceName, nodes = [], focus = false, {source.diagnostics?.affected_node_count > 0 && (

{t(unmappedPd ? 'operations.diagnostic_pd_affected_endpoint' : 'operations.diagnostic_affected_nodes')}: {source.diagnostics.affected_node_count} - {' · '}{t('operations.diagnostic_failed_groups')}: { + {' · '}{t(unmappedPd ? 'operations.diagnostic_unattributed_groups' + : 'operations.diagnostic_failed_groups')}: { source.diagnostics.failed_metric_groups?.map(group => t(`operations.metric_${group}`, {defaultValue: group})).join(', ') || t('operations.unavailable')} diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.test.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.test.js index 5ab70d6c6..16eeaa3df 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.test.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/SourceDiagnostics.test.js @@ -19,12 +19,48 @@ import {fireEvent, render, screen, within} from '@testing-library/react'; import SourceDiagnostics from './SourceDiagnostics'; import i18n from '../../i18n'; +import JSONbig from 'json-bigint'; beforeEach(() => { i18n.changeLanguage('en-US'); window.matchMedia = () => ({matches: false, addListener() {}, removeListener() {}}); }); +test('shows an unmatched PD endpoint sample without assigning it to a member', () => { + const metrics = JSONbig.parse('{"process_cpu_usage":0.00044612345678912345,' + + '"heap_used_bytes":1048576,"threads_live":8}'); + render(); + fireEvent.click(screen.getByText('Check addresses and how to fix')); + expect(screen.getByText('Configured PD endpoint sample')).toBeVisible(); + expect(screen.getByText(/do not identify a node or represent a cluster total/)).toBeVisible(); + expect(screen.getByText('0.04%')).toBeVisible(); + expect(screen.getByText('1 MB')).toBeVisible(); + expect(screen.getByText(/PD endpoints without node attribution: 1/)).toBeVisible(); + expect(screen.getByText(/Unattributed metric groups: System/)).toBeVisible(); + expect(screen.queryByText(/Failed metric groups/)).not.toBeInTheDocument(); + expect(screen.queryByText(/PD A · system/)).not.toBeInTheDocument(); +}); + +test.each([ + ['stale source', true, {availability: 'AVAILABLE', fresh: true}], + ['stale sample', false, {availability: 'AVAILABLE', fresh: false, stale: true}], + ['failed sample', false, {availability: 'UNAVAILABLE', fresh: false, reason: 'upstream_unavailable'}], +])('%s does not display old endpoint values as current', (_, stale, status) => { + render(); + fireEvent.click(screen.getByText('Check addresses and how to fix')); + expect(screen.getByText('Configured PD endpoint sample')).toBeVisible(); + expect(screen.queryByText('12.5%')).not.toBeInTheDocument(); +}); + test('keeps source diagnostics available without discovered nodes and reports bounded response evidence', () => { render(); fireEvent.click(screen.getByText('Check addresses and how to fix')); - expect(screen.getByText(/Affected PD collection endpoints: 1/)).toBeVisible(); + expect(screen.getByText(/PD endpoints without node attribution: 1/)).toBeVisible(); expect(screen.getByText(/Hubble collects system metrics from only the one matched PD member/)).toBeVisible(); expect(screen.queryByText(/Affected nodes across this snapshot/)).not.toBeInTheDocument(); for (const id of [1, 2, 3]) { @@ -159,7 +195,7 @@ test.each(['UNAVAILABLE', 'UNSUPPORTED'])( fireEvent.click(screen.getByText('Check addresses and how to fix')); expect(screen.getByText(/PD topology was discovered.*restUrl in the PD member list/)).toBeInTheDocument(); expect(screen.getByText('Last topology success')).toBeInTheDocument(); - expect(screen.getByText('System metrics not collected')).toBeInTheDocument(); + expect(screen.getByText('Node system metrics not collected')).toBeInTheDocument(); for (const id of [1, 2, 3]) { expect(screen.getByText(new RegExp(`PD ${id} · system`))).toBeVisible(); } diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/VermeerOverview.test.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/VermeerOverview.test.js index 10672d382..6498aadd5 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/VermeerOverview.test.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/VermeerOverview.test.js @@ -18,6 +18,7 @@ import {render, screen, waitFor, within} from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import {MemoryRouter, useLocation, useNavigate} from 'react-router-dom'; +import JSONbig from 'json-bigint'; import Overview from './Overview'; import {getOverview} from '../../api/operations'; import {getDashboard} from '../../api/auth'; @@ -399,6 +400,43 @@ test('resource summary counts only fresh process samples and states coverage per }); }); +test('production json-bigint decimals count as fresh CPU samples without changing zero', async () => { + const data = snapshot(2); + const parsed = JSONbig.parse('{"cpu_cores":0.00044612345678912345,"rss_bytes":1288490188}'); + expect(typeof parsed.cpu_cores).toBe('object'); + data.vermeer.master = {...data.vermeer.master, metrics: {cpu_cores: 0, rss_bytes: 1024}, + metric_status: {availability: 'AVAILABLE', fresh: true}}; + data.vermeer.workers[0].metrics = parsed; + data.vermeer.workers[1].metrics = {cpu_cores: 0, rss_bytes: 2048}; + const summary = vermeerResourceSummary([data.vermeer.master, ...data.vermeer.workers], data.vermeer); + expect(summary.cpu.collected).toBe(3); + expect(summary.cpu.value).toBeCloseTo(Number('0.00044612345678912345')); + getOverview.mockResolvedValue(data); + renderOverview('/operations/overview?service=vermeer&node=worker-1'); + const dialog = await screen.findByRole('dialog'); + expect(within(dialog).getByText('Process CPU: 0.0004 cores')).toBeInTheDocument(); + expect(within(dialog).queryByText('Process CPU: Unavailable')).not.toBeInTheDocument(); +}); + +test.each([null, '0.5', {}, -1, Infinity, NaN])('invalid CPU value %p is unavailable', value => { + const data = snapshot(); + const node = {...data.vermeer.workers[0], metrics: {cpu_cores: value}}; + expect(vermeerResourceSummary([node], data.vermeer).cpu).toEqual({value: null, collected: 0}); +}); + +test('BigNumber metrics still obey availability, freshness and finite nonnegative bounds', () => { + const data = snapshot(); + const big = JSONbig.parse('{"small":0.00044612345678912345,"negative":-0.00044612345678912345}'); + const node = {...data.vermeer.workers[0], metrics: {cpu_cores: big.small}, + metric_status: {availability: 'AVAILABLE', stale: true}}; + expect(vermeerResourceSummary([node], data.vermeer).cpu).toEqual({value: null, collected: 0}); + node.metric_status = {availability: 'UNAVAILABLE'}; + expect(vermeerResourceSummary([node], data.vermeer).cpu).toEqual({value: null, collected: 0}); + node.metric_status = {availability: 'AVAILABLE'}; + node.metrics.cpu_cores = big.negative; + expect(vermeerResourceSummary([node], data.vermeer).cpu).toEqual({value: null, collected: 0}); +}); + test.each([ ['RSS-only', {rss_bytes: 2048}, {cpu: {value: 0.12, collected: 1}, rss: {value: 3072, collected: 2}}], ['CPU-only', {cpu_cores: 0.42}, {cpu: {value: 0.54, collected: 2}, rss: {value: 1024, collected: 1}}], diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/topology.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/topology.js index 092affb10..43acb3d17 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/topology.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/topology.js @@ -23,6 +23,9 @@ const STATUS_PRIORITY = { UP: 3, }; +const markSourcesStale = sources => sources && Object.fromEntries(Object.entries(sources) + .map(([name, source]) => [name, source && {...source, fresh: false, stale: true}])); + const hasStaleMetrics = node => Object.values(node?.metric_statuses ?? {}) .some(status => status?.stale === true); @@ -192,6 +195,7 @@ const formatObservedAge = (value, locale, unavailable, now = Date.now()) => { }; export { + markSourcesStale, selectTierNodes, selectAttentionNodes, selectAttentionSources, diff --git a/hugegraph-hubble/hubble-fe/src/pages/Operations/vermeer.js b/hugegraph-hubble/hubble-fe/src/pages/Operations/vermeer.js index 6f773eb1b..ca2a548dc 100644 --- a/hugegraph-hubble/hubble-fe/src/pages/Operations/vermeer.js +++ b/hugegraph-hubble/hubble-fe/src/pages/Operations/vermeer.js @@ -64,9 +64,20 @@ const vermeerSelectedNode = (nodes, selectedId) => { return previousMatches.length === 1 ? previousMatches[0] : null; }; +// JSON responses use json-bigint, which returns BigNumber for fractional +// values that cannot round-trip through a JavaScript number. These process +// metrics are approximate display values, not identifiers or counters. +const metricNumber = value => { + const bigNumber = value !== null && typeof value === 'object' + && typeof value.constructor?.isBigNumber === 'function' + && value.constructor.isBigNumber(value) && typeof value.toNumber === 'function'; + const number = bigNumber ? value.toNumber() : value; + return typeof number === 'number' && Number.isFinite(number) && number >= 0 ? number : null; +}; + const processMetric = (value, locale) => ( typeof value === 'number' && Number.isFinite(value) && value >= 0 - ? new Intl.NumberFormat(locale, {maximumFractionDigits: 2}).format(value) : null + ? new Intl.NumberFormat(locale, {maximumFractionDigits: 4}).format(value) : null ); const vermeerResourceValues = (node, cluster) => { @@ -74,8 +85,7 @@ const vermeerResourceValues = (node, cluster) => { || node.metric_status?.fresh === false); const available = vermeerAvailable(cluster) && ['AVAILABLE', 'PARTIAL'].includes(node.metric_status?.availability) && !stale; - const metric = value => available && typeof value === 'number' - && Number.isFinite(value) && value >= 0 ? value : null; + const metric = value => available ? metricNumber(value) : null; return {cpu: metric(node.metrics?.cpu_cores), rss: metric(node.metrics?.rss_bytes), stale}; }; @@ -106,6 +116,7 @@ const formatProcessBytes = (bytes, locale) => { export { formatProcessBytes, + metricNumber, processMetric, vermeerAvailable, vermeerHealth,