PySparkAnomalyDetection - Details for Job 212

Status: SUCCEEDED
Submitted: 2026/03/02 11:41:08
Duration: 0.7 s
Associated SQL Query: 425
Job Group: 0c192fb1-cd04-43f7-a1ca-5b186c80fad6
Completed Stages: 3

Enable zooming

digraph G { id="graph_636"; subgraph graph_stage_636 { id="graph_stage_636"; isCluster="true"; label="Stage 636"; subgraph cluster_6638 { id="cluster_6638"; isCluster="true"; label="MicroBatchScan"; 2832 [id="node_2832" labelType="html" label="DataSourceRDD [2832] start at NativeMethodAccessorImpl.java:0}"]; 2833 [id="node_2833" labelType="html" label="MapPartitionsRDD [2833] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6635 { id="cluster_6635"; isCluster="true"; label="WholeStageCodegen (1)"; 2834 [id="node_2834" labelType="html" label="MapPartitionsRDD [2834] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6634 { id="cluster_6634"; isCluster="true"; label="ArrowEvalPython"; 2835 [id="node_2835" labelType="html" label="MapPartitionsRDD [2835] start at NativeMethodAccessorImpl.java:0}"]; 2836 [id="node_2836" labelType="html" label="MapPartitionsRDD [2836] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6630 { id="cluster_6630"; isCluster="true"; label="WholeStageCodegen (2)"; 2837 [id="node_2837" labelType="html" label="MapPartitionsRDD [2837] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6629 { id="cluster_6629"; isCluster="true"; label="ArrowEvalPython"; 2838 [id="node_2838" labelType="html" label="MapPartitionsRDD [2838] start at NativeMethodAccessorImpl.java:0}"]; 2839 [id="node_2839" labelType="html" label="MapPartitionsRDD [2839] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6623 { id="cluster_6623"; isCluster="true"; label="WholeStageCodegen (3)"; 2840 [id="node_2840" labelType="html" label="MapPartitionsRDD [2840] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6622 { id="cluster_6622"; isCluster="true"; label="EventTimeWatermark"; 2841 [id="node_2841" labelType="html" label="MapPartitionsRDD [2841] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6618 { id="cluster_6618"; isCluster="true"; label="WholeStageCodegen (4)"; 2842 [id="node_2842" labelType="html" label="MapPartitionsRDD [2842] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6617 { id="cluster_6617"; isCluster="true"; label="ObjectHashAggregate"; 2843 [id="node_2843" labelType="html" label="MapPartitionsRDD [2843] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6616 { id="cluster_6616"; isCluster="true"; label="Exchange"; 2844 [id="node_2844" labelType="html" label="MapPartitionsRDD [2844] start at NativeMethodAccessorImpl.java:0}"]; } } 2832->2833; 2833->2834; 2834->2835; 2835->2836; 2836->2837; 2837->2838; 2838->2839; 2839->2840; 2840->2841; 2841->2842; 2842->2843; 2843->2844; }

digraph G { id="graph_637"; subgraph graph_stage_637 { id="graph_stage_637"; isCluster="true"; label="Stage 637"; 2852 [id="node_2852" labelType="html" label="SQLExecutionRDD [2852] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; subgraph cluster_6616 { id="cluster_6616"; isCluster="true"; label="Exchange"; 2845 [id="node_2845" labelType="html" label="ShuffledRowRDD [2845] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6615 { id="cluster_6615"; isCluster="true"; label="ObjectHashAggregate"; 2846 [id="node_2846" labelType="html" label="MapPartitionsRDD [2846] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6614 { id="cluster_6614"; isCluster="true"; label="StateStoreRestore"; 2847 [id="node_2847" labelType="html" label="ReadStateStoreRDD [2847] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6613 { id="cluster_6613"; isCluster="true"; label="ObjectHashAggregate"; 2848 [id="node_2848" labelType="html" label="MapPartitionsRDD [2848] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6612 { id="cluster_6612"; isCluster="true"; label="StateStoreSave"; 2849 [id="node_2849" labelType="html" label="StateStoreRDD [2849] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6611 { id="cluster_6611"; isCluster="true"; label="ObjectHashAggregate"; 2850 [id="node_2850" labelType="html" label="MapPartitionsRDD [2850] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6610 { id="cluster_6610"; isCluster="true"; label="Project"; 2851 [id="node_2851" labelType="html" label="MapPartitionsRDD [2851] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6640 { id="cluster_6640"; isCluster="true"; label="WholeStageCodegen (1)"; 2853 [id="node_2853" labelType="html" label="MapPartitionsRDD [2853] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6639 { id="cluster_6639"; isCluster="true"; label="CollectLimit"; 2854 [id="node_2854" labelType="html" label="MapPartitionsRDD [2854] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 2855 [id="node_2855" labelType="html" label="MapPartitionsRDD [2855] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 2845->2846; 2846->2847; 2847->2848; 2848->2849; 2849->2850; 2850->2851; 2851->2852; 2852->2853; 2853->2854; 2854->2855; }

2844,2845

digraph G { id="graph_638"; subgraph graph_stage_638 { id="graph_stage_638"; isCluster="true"; label="Stage 638"; subgraph cluster_6639 { id="cluster_6639"; isCluster="true"; label="CollectLimit"; 2856 [id="node_2856" labelType="html" label="ShuffledRowRDD [2856] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; 2857 [id="node_2857" labelType="html" label="MapPartitionsRDD [2857] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } subgraph cluster_6643 { id="cluster_6643"; isCluster="true"; label="mapPartitionsInternal"; 2858 [id="node_2858" labelType="html" label="MapPartitionsRDD [2858] [Unordered] start at NativeMethodAccessorImpl.java:0}"]; } } 2856->2857; 2857->2858; }

2855,2856

Completed Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total
638	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:09	7 ms	1/1
637	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:09	0.1 s	2/2
636	id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) java.base/java.lang.reflect.Method.invoke(Method.java:580) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184) py4j.ClientServerConnection.run(ClientServerConnection.java:108) java.base/java.lang.Thread.run(Thread.java:1583)	2026/03/02 11:41:08	0.5 s	1/1

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

638

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:09

7 ms

1/1

637

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:09

0.1 s

2/2

636

id = 7c48e2f8-44c2-4bf3-b908-af48ebd97a06 runId = 0c192fb1-cd04-43f7-a1ca-5b186c80fad6 batch = 15 start at NativeMethodAccessorImpl.java:0 +details

org.apache.spark.sql.classic.DataStreamWriter.start(DataStreamWriter.scala:137)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:75)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52)
java.base/java.lang.reflect.Method.invoke(Method.java:580)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:184)
py4j.ClientServerConnection.run(ClientServerConnection.java:108)
java.base/java.lang.Thread.run(Thread.java:1583)

2026/03/02 11:41:08

0.5 s

1/1

Page: