PySparkAnomalyDetection - Details for Job 267

Status: SUCCEEDED
Submitted: 2026/03/02 11:41:47
Duration: 0.8 s
Associated SQL Query: 535
Job Group: 0c192fb1-cd04-43f7-a1ca-5b186c80fad6
Completed Stages: 3

Enable zooming

digraph G { id="graph_801"; subgraph graph_stage_801 { id="graph_stage_801"; isCluster="true"; label="Stage 801"; subgraph cluster_8328 { id="cluster_8328"; isCluster="true"; label="MicroBatchScan"; 3551 [id="node_3551" labelType="html" label="DataSourceRDD [3551] start at NativeMethodAccessorImpl.java:0}"]; 3552 [id="node_3552" labelType="html" label="MapPartitionsRDD [3552] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8325 { id="cluster_8325"; isCluster="true"; label="WholeStageCodegen (1)"; 3553 [id="node_3553" labelType="html" label="MapPartitionsRDD [3553] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8324 { id="cluster_8324"; isCluster="true"; label="ArrowEvalPython"; 3554 [id="node_3554" labelType="html" label="MapPartitionsRDD [3554] start at NativeMethodAccessorImpl.java:0}"]; 3555 [id="node_3555" labelType="html" label="MapPartitionsRDD [3555] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8320 { id="cluster_8320"; isCluster="true"; label="WholeStageCodegen (2)"; 3556 [id="node_3556" labelType="html" label="MapPartitionsRDD [3556] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8319 { id="cluster_8319"; isCluster="true"; label="ArrowEvalPython"; 3557 [id="node_3557" labelType="html" label="MapPartitionsRDD [3557] start at NativeMethodAccessorImpl.java:0}"]; 3558 [id="node_3558" labelType="html" label="MapPartitionsRDD [3558] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8313 { id="cluster_8313"; isCluster="true"; label="WholeStageCodegen (3)"; 3559 [id="node_3559" labelType="html" label="MapPartitionsRDD [3559] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8312 { id="cluster_8312"; isCluster="true"; label="EventTimeWatermark"; 3560 [id="node_3560" labelType="html" label="MapPartitionsRDD [3560] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8308 { id="cluster_8308"; isCluster="true"; label="WholeStageCodegen (4)"; 3561 [id="node_3561" labelType="html" label="MapPartitionsRDD [3561] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8307 { id="cluster_8307"; isCluster="true"; label="ObjectHashAggregate"; 3562 [id="node_3562" labelType="html" label="MapPartitionsRDD [3562] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8306 { id="cluster_8306"; isCluster="true"; label="Exchange"; 3563 [id="node_3563" labelType="html" label="MapPartitionsRDD [3563] start at NativeMethodAccessorImpl.java:0}"]; } } 3551->3552; 3552->3553; 3553->3554; 3554->3555; 3555->3556; 3556->3557; 3557->3558; 3558->3559; 3559->3560; 3560->3561; 3561->3562; 3562->3563; }

digraph G { id="graph_802"; subgraph graph_stage_802 { id="graph_stage_802"; isCluster="true"; label="Stage 802"; 3571 [id="node_3571" labelType="html" label="SQLExecutionRDD [3571] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; subgraph cluster_8306 { id="cluster_8306"; isCluster="true"; label="Exchange"; 3564 [id="node_3564" labelType="html" label="ShuffledRowRDD [3564] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8305 { id="cluster_8305"; isCluster="true"; label="ObjectHashAggregate"; 3565 [id="node_3565" labelType="html" label="MapPartitionsRDD [3565] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8304 { id="cluster_8304"; isCluster="true"; label="StateStoreRestore"; 3566 [id="node_3566" labelType="html" label="ReadStateStoreRDD [3566] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8303 { id="cluster_8303"; isCluster="true"; label="ObjectHashAggregate"; 3567 [id="node_3567" labelType="html" label="MapPartitionsRDD [3567] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8302 { id="cluster_8302"; isCluster="true"; label="StateStoreSave"; 3568 [id="node_3568" labelType="html" label="StateStoreRDD [3568] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8301 { id="cluster_8301"; isCluster="true"; label="ObjectHashAggregate"; 3569 [id="node_3569" labelType="html" label="MapPartitionsRDD [3569] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8300 { id="cluster_8300"; isCluster="true"; label="Project"; 3570 [id="node_3570" labelType="html" label="MapPartitionsRDD [3570] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8330 { id="cluster_8330"; isCluster="true"; label="WholeStageCodegen (1)"; 3572 [id="node_3572" labelType="html" label="MapPartitionsRDD [3572] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8329 { id="cluster_8329"; isCluster="true"; label="CollectLimit"; 3573 [id="node_3573" labelType="html" label="MapPartitionsRDD [3573] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 3574 [id="node_3574" labelType="html" label="MapPartitionsRDD [3574] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 3564->3565; 3565->3566; 3566->3567; 3567->3568; 3568->3569; 3569->3570; 3570->3571; 3571->3572; 3572->3573; 3573->3574; }

3563,3564

digraph G { id="graph_803"; subgraph graph_stage_803 { id="graph_stage_803"; isCluster="true"; label="Stage 803"; subgraph cluster_8329 { id="cluster_8329"; isCluster="true"; label="CollectLimit"; 3575 [id="node_3575" labelType="html" label="ShuffledRowRDD [3575] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 3576 [id="node_3576" labelType="html" label="MapPartitionsRDD [3576] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_8333 { id="cluster_8333"; isCluster="true"; label="mapPartitionsInternal"; 3577 [id="node_3577" labelType="html" label="MapPartitionsRDD [3577] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 3575->3576; 3576->3577; }

3574,3575

Completed Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
803	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:48	5 ms	1/1
802	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:48	0.2 s	2/2
801	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:47	0.5 s	1/1

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

803

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:48

5 ms

1/1

802

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:48

0.2 s

2/2

801

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 18 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:47

0.5 s

1/1

Page: