PySparkAnomalyDetection - Details for Job 241

Status: SUCCEEDED
Submitted: 2026/03/02 11:41:28
Duration: 0.9 s
Associated SQL Query: 483
Job Group: 0c192fb1-cd04-43f7-a1ca-5b186c80fad6
Completed Stages: 3

Enable zooming

digraph G { id="graph_723"; subgraph graph_stage_723 { id="graph_stage_723"; isCluster="true"; label="Stage 723"; subgraph cluster_7538 { id="cluster_7538"; isCluster="true"; label="MicroBatchScan"; 3217 [id="node_3217" labelType="html" label="DataSourceRDD [3217] start at NativeMethodAccessorImpl.java:0}"]; 3218 [id="node_3218" labelType="html" label="MapPartitionsRDD [3218] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7535 { id="cluster_7535"; isCluster="true"; label="WholeStageCodegen (1)"; 3219 [id="node_3219" labelType="html" label="MapPartitionsRDD [3219] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7534 { id="cluster_7534"; isCluster="true"; label="ArrowEvalPython"; 3220 [id="node_3220" labelType="html" label="MapPartitionsRDD [3220] start at NativeMethodAccessorImpl.java:0}"]; 3221 [id="node_3221" labelType="html" label="MapPartitionsRDD [3221] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7530 { id="cluster_7530"; isCluster="true"; label="WholeStageCodegen (2)"; 3222 [id="node_3222" labelType="html" label="MapPartitionsRDD [3222] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7529 { id="cluster_7529"; isCluster="true"; label="ArrowEvalPython"; 3223 [id="node_3223" labelType="html" label="MapPartitionsRDD [3223] start at NativeMethodAccessorImpl.java:0}"]; 3224 [id="node_3224" labelType="html" label="MapPartitionsRDD [3224] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7523 { id="cluster_7523"; isCluster="true"; label="WholeStageCodegen (3)"; 3225 [id="node_3225" labelType="html" label="MapPartitionsRDD [3225] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7522 { id="cluster_7522"; isCluster="true"; label="EventTimeWatermark"; 3226 [id="node_3226" labelType="html" label="MapPartitionsRDD [3226] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7518 { id="cluster_7518"; isCluster="true"; label="WholeStageCodegen (4)"; 3227 [id="node_3227" labelType="html" label="MapPartitionsRDD [3227] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7517 { id="cluster_7517"; isCluster="true"; label="ObjectHashAggregate"; 3228 [id="node_3228" labelType="html" label="MapPartitionsRDD [3228] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7516 { id="cluster_7516"; isCluster="true"; label="Exchange"; 3229 [id="node_3229" labelType="html" label="MapPartitionsRDD [3229] start at NativeMethodAccessorImpl.java:0}"]; } } 3217->3218; 3218->3219; 3219->3220; 3220->3221; 3221->3222; 3222->3223; 3223->3224; 3224->3225; 3225->3226; 3226->3227; 3227->3228; 3228->3229; }

digraph G { id="graph_724"; subgraph graph_stage_724 { id="graph_stage_724"; isCluster="true"; label="Stage 724"; 3237 [id="node_3237" labelType="html" label="SQLExecutionRDD [3237] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; subgraph cluster_7516 { id="cluster_7516"; isCluster="true"; label="Exchange"; 3230 [id="node_3230" labelType="html" label="ShuffledRowRDD [3230] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7515 { id="cluster_7515"; isCluster="true"; label="ObjectHashAggregate"; 3231 [id="node_3231" labelType="html" label="MapPartitionsRDD [3231] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7514 { id="cluster_7514"; isCluster="true"; label="StateStoreRestore"; 3232 [id="node_3232" labelType="html" label="ReadStateStoreRDD [3232] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7513 { id="cluster_7513"; isCluster="true"; label="ObjectHashAggregate"; 3233 [id="node_3233" labelType="html" label="MapPartitionsRDD [3233] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7512 { id="cluster_7512"; isCluster="true"; label="StateStoreSave"; 3234 [id="node_3234" labelType="html" label="StateStoreRDD [3234] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7511 { id="cluster_7511"; isCluster="true"; label="ObjectHashAggregate"; 3235 [id="node_3235" labelType="html" label="MapPartitionsRDD [3235] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7510 { id="cluster_7510"; isCluster="true"; label="Project"; 3236 [id="node_3236" labelType="html" label="MapPartitionsRDD [3236] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7540 { id="cluster_7540"; isCluster="true"; label="WholeStageCodegen (1)"; 3238 [id="node_3238" labelType="html" label="MapPartitionsRDD [3238] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7539 { id="cluster_7539"; isCluster="true"; label="CollectLimit"; 3239 [id="node_3239" labelType="html" label="MapPartitionsRDD [3239] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 3240 [id="node_3240" labelType="html" label="MapPartitionsRDD [3240] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 3230->3231; 3231->3232; 3232->3233; 3233->3234; 3234->3235; 3235->3236; 3236->3237; 3237->3238; 3238->3239; 3239->3240; }

3229,3230

digraph G { id="graph_725"; subgraph graph_stage_725 { id="graph_stage_725"; isCluster="true"; label="Stage 725"; subgraph cluster_7539 { id="cluster_7539"; isCluster="true"; label="CollectLimit"; 3241 [id="node_3241" labelType="html" label="ShuffledRowRDD [3241] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 3242 [id="node_3242" labelType="html" label="MapPartitionsRDD [3242] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_7543 { id="cluster_7543"; isCluster="true"; label="mapPartitionsInternal"; 3243 [id="node_3243" labelType="html" label="MapPartitionsRDD [3243] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 3241->3242; 3242->3243; }

3240,3241

Completed Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
725	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:29	6 ms	1/1
724	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:29	0.3 s	2/2
723	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:28	0.5 s	2/2

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

725

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:29

6 ms

1/1

724

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:29

0.3 s

2/2

723

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 17 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:28

0.5 s

2/2

Page: