PySparkAnomalyDetection - Details for Job 210

Status: SUCCEEDED
Submitted: 2026/03/02 11:41:07
Duration: 0.7 s
Associated SQL Query: 421
Job Group: 0c192fb1-cd04-43f7-a1ca-5b186c80fad6
Completed Stages: 3

Enable zooming

digraph G { id="graph_630"; subgraph graph_stage_630 { id="graph_stage_630"; isCluster="true"; label="Stage 630"; subgraph cluster_6546 { id="cluster_6546"; isCluster="true"; label="MicroBatchScan"; 2786 [id="node_2786" labelType="html" label="DataSourceRDD [2786] start at NativeMethodAccessorImpl.java:0}"]; 2787 [id="node_2787" labelType="html" label="MapPartitionsRDD [2787] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6543 { id="cluster_6543"; isCluster="true"; label="WholeStageCodegen (1)"; 2788 [id="node_2788" labelType="html" label="MapPartitionsRDD [2788] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6542 { id="cluster_6542"; isCluster="true"; label="ArrowEvalPython"; 2789 [id="node_2789" labelType="html" label="MapPartitionsRDD [2789] start at NativeMethodAccessorImpl.java:0}"]; 2790 [id="node_2790" labelType="html" label="MapPartitionsRDD [2790] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6538 { id="cluster_6538"; isCluster="true"; label="WholeStageCodegen (2)"; 2791 [id="node_2791" labelType="html" label="MapPartitionsRDD [2791] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6537 { id="cluster_6537"; isCluster="true"; label="ArrowEvalPython"; 2792 [id="node_2792" labelType="html" label="MapPartitionsRDD [2792] start at NativeMethodAccessorImpl.java:0}"]; 2793 [id="node_2793" labelType="html" label="MapPartitionsRDD [2793] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6531 { id="cluster_6531"; isCluster="true"; label="WholeStageCodegen (3)"; 2794 [id="node_2794" labelType="html" label="MapPartitionsRDD [2794] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6530 { id="cluster_6530"; isCluster="true"; label="EventTimeWatermark"; 2795 [id="node_2795" labelType="html" label="MapPartitionsRDD [2795] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6526 { id="cluster_6526"; isCluster="true"; label="WholeStageCodegen (4)"; 2796 [id="node_2796" labelType="html" label="MapPartitionsRDD [2796] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6525 { id="cluster_6525"; isCluster="true"; label="ObjectHashAggregate"; 2797 [id="node_2797" labelType="html" label="MapPartitionsRDD [2797] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6524 { id="cluster_6524"; isCluster="true"; label="Exchange"; 2798 [id="node_2798" labelType="html" label="MapPartitionsRDD [2798] start at NativeMethodAccessorImpl.java:0}"]; } } 2786->2787; 2787->2788; 2788->2789; 2789->2790; 2790->2791; 2791->2792; 2792->2793; 2793->2794; 2794->2795; 2795->2796; 2796->2797; 2797->2798; }

digraph G { id="graph_631"; subgraph graph_stage_631 { id="graph_stage_631"; isCluster="true"; label="Stage 631"; 2806 [id="node_2806" labelType="html" label="SQLExecutionRDD [2806] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; subgraph cluster_6524 { id="cluster_6524"; isCluster="true"; label="Exchange"; 2799 [id="node_2799" labelType="html" label="ShuffledRowRDD [2799] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6523 { id="cluster_6523"; isCluster="true"; label="ObjectHashAggregate"; 2800 [id="node_2800" labelType="html" label="MapPartitionsRDD [2800] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6522 { id="cluster_6522"; isCluster="true"; label="StateStoreRestore"; 2801 [id="node_2801" labelType="html" label="ReadStateStoreRDD [2801] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6521 { id="cluster_6521"; isCluster="true"; label="ObjectHashAggregate"; 2802 [id="node_2802" labelType="html" label="MapPartitionsRDD [2802] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6520 { id="cluster_6520"; isCluster="true"; label="StateStoreSave"; 2803 [id="node_2803" labelType="html" label="StateStoreRDD [2803] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6519 { id="cluster_6519"; isCluster="true"; label="ObjectHashAggregate"; 2804 [id="node_2804" labelType="html" label="MapPartitionsRDD [2804] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6518 { id="cluster_6518"; isCluster="true"; label="Project"; 2805 [id="node_2805" labelType="html" label="MapPartitionsRDD [2805] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6548 { id="cluster_6548"; isCluster="true"; label="WholeStageCodegen (1)"; 2807 [id="node_2807" labelType="html" label="MapPartitionsRDD [2807] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6547 { id="cluster_6547"; isCluster="true"; label="CollectLimit"; 2808 [id="node_2808" labelType="html" label="MapPartitionsRDD [2808] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 2809 [id="node_2809" labelType="html" label="MapPartitionsRDD [2809] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 2799->2800; 2800->2801; 2801->2802; 2802->2803; 2803->2804; 2804->2805; 2805->2806; 2806->2807; 2807->2808; 2808->2809; }

2798,2799

digraph G { id="graph_632"; subgraph graph_stage_632 { id="graph_stage_632"; isCluster="true"; label="Stage 632"; subgraph cluster_6547 { id="cluster_6547"; isCluster="true"; label="CollectLimit"; 2810 [id="node_2810" labelType="html" label="ShuffledRowRDD [2810] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 2811 [id="node_2811" labelType="html" label="MapPartitionsRDD [2811] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6551 { id="cluster_6551"; isCluster="true"; label="mapPartitionsInternal"; 2812 [id="node_2812" labelType="html" label="MapPartitionsRDD [2812] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 2810->2811; 2811->2812; }

2809,2810

Completed Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
632	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:08	5 ms	1/1
631	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:08	95 ms	2/2
630	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:07	0.6 s	1/1

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

632

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:08

5 ms

1/1

631

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:08

95 ms

2/2

630

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 14 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:07

0.6 s

1/1

Page: