{"Event":"SparkListenerLogStart","Spark Version":"3.5.1"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"73850a3d530b","Port":41051},"Maximum Memory":455501414,"Timestamp":1784293422627,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.19 (Debian)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"73850a3d530b","spark.serializer.objectStreamReset":"100","spark.eventLog.enabled":"true","spark.driver.port":"33853","spark.rdd.compress":"True","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.name":"anazarenko_create_dm_gp","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.submitTime":"1784293421026","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.submit.deployMode":"client","spark.master":"spark://spark-master:7077","spark.eventLog.dir":"file:/opt/spark-events","spark.eventLog.compress":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.id":"app-20260717170342-0094"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n    org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n    org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n  ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"1","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Debian","java.vm.specification.version":"17","user.home":"/home/airflow","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-17-openjdk-amd64/lib","user.dir":"/opt/airflow","java.library.path":"/usr/local/lib:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.19+10-1-deb12u2-Debian","jetty.git.hash":"abdcda73818a1a2c705da276edb0bf6581e7997e","java.runtime.version":"17.0.19+10-1-deb12u2-Debian","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2026-04-21","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Europe/Samara","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.12.48+deb13-amd64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://tracker.debian.org/openjdk-17","os.name":"Linux","java.vm.vendor":"Debian","jdk.debug":"release","java.vendor.url.bug":"https://bugs.debian.org/openjdk-17","user.name":"airflow","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://spark-master:7077 --deploy-mode client --conf spark.eventLog.compress=true --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=file:/opt/spark-events --packages org.postgresql:postgresql:42.7.3 --name airflow-spark --verbose /opt/airflow/dags/anazarenko/spark/create_dm_gp.py --input s3a://anazarenko-bucket/input/ --output s3a://anazarenko-bucket/output/","java.home":"/usr/lib/jvm/java-17-openjdk-amd64","java.version":"17.0.19","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snappy-java-1.1.10.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JLargeArrays-1.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-logging-1.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/super-csv-2.2.0.jar":"System Classpath","spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compiler-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sketch_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-common-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-format-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xz-1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-launcher_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-aws-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill-java-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-server-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/tink-1.9.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-core_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-library-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-metastore-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lz4-java-1.8.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/conf":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-tags_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive-thriftserver_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/derby-10.14.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-api_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/gson-2.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/activation-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jta-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-mapreduce-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpclient-4.5.14.jar":"System Classpath","spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill_2.12-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jdo-api-3.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-math3-3.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool2-2.11.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/oro-2.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-cli-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-core-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-jdbc-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang3-3.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections4-4.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/postgresql-42.7.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kubernetes_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-token-provider-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool-1.5.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-column-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jsr305-3.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-json-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/threeten-extra-1.7.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javolution-5.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/shims-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections-3.2.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jpam-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-shuffle_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/slf4j-api-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libthrift-0.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-repl_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-compiler-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/minlog-1.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/paranamer-2.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-client-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-vector-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-core-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aircompressor-0.26.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/guava-14.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JTransforms-3.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/py4j-0.10.9.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/pickle-1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/opencsv-2.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-databind-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/HikariCP-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ST4-4.0.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-dbcp-1.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-client-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpcore-4.4.16.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/joda-time-2.12.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/objenesis-3.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-yarn_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-beeline-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-catalyst_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib-local_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/annotations-17.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aws-java-sdk-bundle-1.12.262.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javassist-3.29.2-GA.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-text-1.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/compress-lzf-1.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-ipc-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-io-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ivy-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-streaming_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okhttp-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-mapred-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-common-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-cli-1.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/clickhouse-jdbc-0.6.0-all.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libfb303-0.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stream-2.9.6.jar":"System Classpath","spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-reflect-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-api-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-java-3.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okio-1.15.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-common_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kafka-clients-3.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-common-utils_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lapack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-core-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/janino-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/blas-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-graphx_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jodd-core-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mesos_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-hk2-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compress-1.23.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stax-api-1.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-serde-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-all-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-shims-1.9.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-framework-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kvstore_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-codec-1.16.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jline-2.14.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-unsafe_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/transaction-api-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"anazarenko_create_dm_gp","App ID":"app-20260717170342-0094","Timestamp":1784293421815,"User":"airflow"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1784293424560,"Executor ID":"0","Executor Info":{"Host":"172.28.0.49","Total Cores":5,"Log Urls":{"stdout":"http://172.28.0.49:8081/logPage/?appId=app-20260717170342-0094&executorId=0&logType=stdout","stderr":"http://172.28.0.49:8081/logPage/?appId=app-20260717170342-0094&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1784293424560}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.49","Port":41697},"Maximum Memory":455501414,"Timestamp":1784293424652,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- CollectLimit (42)\n   +- Project (41)\n      +- SortMergeJoin LeftOuter (40)\n         :- LocalLimit (23)\n         :  +- Project (22)\n         :     +- SortMergeJoin LeftOuter (21)\n         :        :- LocalLimit (12)\n         :        :  +- Project (11)\n         :        :     +- SortMergeJoin Inner (10)\n         :        :        :- Sort (4)\n         :        :        :  +- Exchange (3)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (9)\n         :        :           +- HashAggregate (8)\n         :        :              +- Exchange (7)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (20)\n         :           +- HashAggregate (19)\n         :              +- Exchange (18)\n         :                 +- HashAggregate (17)\n         :                    +- HashAggregate (16)\n         :                       +- Exchange (15)\n         :                          +- HashAggregate (14)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (13)\n         +- Sort (39)\n            +- HashAggregate (38)\n               +- Exchange (37)\n                  +- HashAggregate (36)\n                     +- HashAggregate (35)\n                        +- Exchange (34)\n                           +- HashAggregate (33)\n                              +- Project (32)\n                                 +- SortMergeJoin Inner (31)\n                                    :- Sort (26)\n                                    :  +- Exchange (25)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (24)\n                                    +- Sort (30)\n                                       +- Exchange (29)\n                                          +- Project (28)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (27)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1] \nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(4) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1] \nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(8) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(9) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(10) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(11) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(12) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(13) Scan JDBCRelation(raw.lesson_views) [numPartitions=1] \nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(14) HashAggregate\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(15) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(16) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(17) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(18) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(19) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(20) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(21) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(23) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(24) Scan JDBCRelation(raw.enrollments) [numPartitions=1] \nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(25) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(26) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(27) Scan JDBCRelation(raw.lessons) [numPartitions=1] \nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(28) Project\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(29) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(30) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(32) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(33) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(34) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(35) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(36) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(37) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(38) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(39) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(40) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(41) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(42) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(43) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#114], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#51], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#43], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":50,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":44,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":51,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":45,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":42,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":39,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":48,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":41,"metricType":"sum"},{"name":"records read","accumulatorId":38,"metricType":"sum"},{"name":"local bytes read","accumulatorId":36,"metricType":"size"},{"name":"fetch wait time","accumulatorId":37,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":34,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":40,"metricType":"sum"},{"name":"local blocks read","accumulatorId":33,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":43,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":32,"metricType":"sum"},{"name":"data size","accumulatorId":30,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":46,"metricType":"size"},{"name":"number of partitions","accumulatorId":31,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":47,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":35,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":49,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":27,"metricType":"timing"},{"name":"peak memory","accumulatorId":28,"metricType":"size"},{"name":"spill size","accumulatorId":29,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#43 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":82,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":76,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":83,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":77,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":74,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":71,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":80,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":73,"metricType":"sum"},{"name":"records read","accumulatorId":70,"metricType":"sum"},{"name":"local bytes read","accumulatorId":68,"metricType":"size"},{"name":"fetch wait time","accumulatorId":69,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":66,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":72,"metricType":"sum"},{"name":"local blocks read","accumulatorId":65,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":75,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":64,"metricType":"sum"},{"name":"data size","accumulatorId":62,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":78,"metricType":"size"},{"name":"number of partitions","accumulatorId":63,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":79,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":67,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":81,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":58,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":59,"metricType":"timing"},{"name":"peak memory","accumulatorId":57,"metricType":"size"},{"name":"number of output rows","accumulatorId":56,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":61,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":60,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":53,"metricType":"timing"},{"name":"peak memory","accumulatorId":54,"metricType":"size"},{"name":"spill size","accumulatorId":55,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":25,"metricType":"sum"},{"name":"spill size","accumulatorId":26,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#51 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":154,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":148,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":155,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":149,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":146,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":143,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":152,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":145,"metricType":"sum"},{"name":"records read","accumulatorId":142,"metricType":"sum"},{"name":"local bytes read","accumulatorId":140,"metricType":"size"},{"name":"fetch wait time","accumulatorId":141,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":138,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":144,"metricType":"sum"},{"name":"local blocks read","accumulatorId":137,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":147,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":136,"metricType":"sum"},{"name":"data size","accumulatorId":134,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":150,"metricType":"size"},{"name":"number of partitions","accumulatorId":135,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":151,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":139,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":153,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":130,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":131,"metricType":"timing"},{"name":"peak memory","accumulatorId":129,"metricType":"size"},{"name":"number of output rows","accumulatorId":128,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":133,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":132,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":124,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":125,"metricType":"timing"},{"name":"peak memory","accumulatorId":123,"metricType":"size"},{"name":"number of output rows","accumulatorId":122,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":127,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":126,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":120,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":114,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":121,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":115,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":112,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":109,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":118,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":111,"metricType":"sum"},{"name":"records read","accumulatorId":108,"metricType":"sum"},{"name":"local bytes read","accumulatorId":106,"metricType":"size"},{"name":"fetch wait time","accumulatorId":107,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":104,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":110,"metricType":"sum"},{"name":"local blocks read","accumulatorId":103,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":113,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":102,"metricType":"sum"},{"name":"data size","accumulatorId":100,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":116,"metricType":"size"},{"name":"number of partitions","accumulatorId":101,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":117,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":105,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":119,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":96,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":97,"metricType":"timing"},{"name":"peak memory","accumulatorId":95,"metricType":"size"},{"name":"number of output rows","accumulatorId":94,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":99,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":98,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":91,"metricType":"timing"},{"name":"peak memory","accumulatorId":92,"metricType":"size"},{"name":"spill size","accumulatorId":93,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":23,"metricType":"sum"},{"name":"spill size","accumulatorId":24,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#114 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":259,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":253,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":260,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":254,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":251,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":248,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":257,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":250,"metricType":"sum"},{"name":"records read","accumulatorId":247,"metricType":"sum"},{"name":"local bytes read","accumulatorId":245,"metricType":"size"},{"name":"fetch wait time","accumulatorId":246,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":243,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":249,"metricType":"sum"},{"name":"local blocks read","accumulatorId":242,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":252,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":241,"metricType":"sum"},{"name":"data size","accumulatorId":239,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":255,"metricType":"size"},{"name":"number of partitions","accumulatorId":240,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":256,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":244,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":258,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":236,"metricType":"timing"},{"name":"peak memory","accumulatorId":237,"metricType":"size"},{"name":"spill size","accumulatorId":238,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":285,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":279,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":286,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":280,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":277,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":274,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":283,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":276,"metricType":"sum"},{"name":"records read","accumulatorId":273,"metricType":"sum"},{"name":"local bytes read","accumulatorId":271,"metricType":"size"},{"name":"fetch wait time","accumulatorId":272,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":269,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":275,"metricType":"sum"},{"name":"local blocks read","accumulatorId":268,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":278,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":267,"metricType":"sum"},{"name":"data size","accumulatorId":265,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":281,"metricType":"size"},{"name":"number of partitions","accumulatorId":266,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":282,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":270,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":284,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":262,"metricType":"timing"},{"name":"peak memory","accumulatorId":263,"metricType":"size"},{"name":"spill size","accumulatorId":264,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":234,"metricType":"sum"},{"name":"spill size","accumulatorId":235,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":230,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":231,"metricType":"timing"},{"name":"peak memory","accumulatorId":229,"metricType":"size"},{"name":"number of output rows","accumulatorId":228,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":233,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":232,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":226,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":220,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":227,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":221,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":218,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":215,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":224,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":217,"metricType":"sum"},{"name":"records read","accumulatorId":214,"metricType":"sum"},{"name":"local bytes read","accumulatorId":212,"metricType":"size"},{"name":"fetch wait time","accumulatorId":213,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":210,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":216,"metricType":"sum"},{"name":"local blocks read","accumulatorId":209,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":219,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":208,"metricType":"sum"},{"name":"data size","accumulatorId":206,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":222,"metricType":"size"},{"name":"number of partitions","accumulatorId":207,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":223,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":211,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":225,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":202,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":203,"metricType":"timing"},{"name":"peak memory","accumulatorId":201,"metricType":"size"},{"name":"number of output rows","accumulatorId":200,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":205,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":204,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":196,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":197,"metricType":"timing"},{"name":"peak memory","accumulatorId":195,"metricType":"size"},{"name":"number of output rows","accumulatorId":194,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":199,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":198,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":192,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":186,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":193,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":187,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":184,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":181,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":190,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":183,"metricType":"sum"},{"name":"records read","accumulatorId":180,"metricType":"sum"},{"name":"local bytes read","accumulatorId":178,"metricType":"size"},{"name":"fetch wait time","accumulatorId":179,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":176,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":182,"metricType":"sum"},{"name":"local blocks read","accumulatorId":175,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":185,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":174,"metricType":"sum"},{"name":"data size","accumulatorId":172,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":188,"metricType":"size"},{"name":"number of partitions","accumulatorId":173,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":189,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":177,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":191,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":168,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":169,"metricType":"timing"},{"name":"peak memory","accumulatorId":167,"metricType":"size"},{"name":"number of output rows","accumulatorId":166,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":171,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":170,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":163,"metricType":"timing"},{"name":"peak memory","accumulatorId":164,"metricType":"size"},{"name":"spill size","accumulatorId":165,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":21,"metricType":"sum"},{"name":"spill size","accumulatorId":22,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":19,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":13,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":20,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":14,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":11,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":8,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":17,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":10,"metricType":"sum"},{"name":"records read","accumulatorId":7,"metricType":"sum"},{"name":"local bytes read","accumulatorId":5,"metricType":"size"},{"name":"fetch wait time","accumulatorId":6,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":9,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":12,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":15,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":16,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":18,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784293425805,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (81)\n+- == Current Plan ==\n   CollectLimit (47)\n   +- Project (46)\n      +- SortMergeJoin LeftOuter (45)\n         :- LocalLimit (26)\n         :  +- Project (25)\n         :     +- SortMergeJoin LeftOuter (24)\n         :        :- LocalLimit (14)\n         :        :  +- Project (13)\n         :        :     +- SortMergeJoin Inner (12)\n         :        :        :- Sort (5)\n         :        :        :  +- ShuffleQueryStage (4)\n         :        :        :     +- Exchange (3)\n         :        :        :        +- * Project (2)\n         :        :        :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (11)\n         :        :           +- HashAggregate (10)\n         :        :              +- ShuffleQueryStage (9)\n         :        :                 +- Exchange (8)\n         :        :                    +- * HashAggregate (7)\n         :        :                       +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         :        +- Sort (23)\n         :           +- HashAggregate (22)\n         :              +- Exchange (21)\n         :                 +- HashAggregate (20)\n         :                    +- HashAggregate (19)\n         :                       +- ShuffleQueryStage (18)\n         :                          +- Exchange (17)\n         :                             +- * HashAggregate (16)\n         :                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (15)\n         +- Sort (44)\n            +- HashAggregate (43)\n               +- Exchange (42)\n                  +- HashAggregate (41)\n                     +- HashAggregate (40)\n                        +- Exchange (39)\n                           +- HashAggregate (38)\n                              +- Project (37)\n                                 +- SortMergeJoin Inner (36)\n                                    :- Sort (30)\n                                    :  +- ShuffleQueryStage (29)\n                                    :     +- Exchange (28)\n                                    :        +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (27)\n                                    +- Sort (35)\n                                       +- ShuffleQueryStage (34)\n                                          +- Exchange (33)\n                                             +- * Project (32)\n                                                +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (31)\n+- == Initial Plan ==\n   CollectLimit (80)\n   +- Project (79)\n      +- SortMergeJoin LeftOuter (78)\n         :- LocalLimit (64)\n         :  +- Project (63)\n         :     +- SortMergeJoin LeftOuter (62)\n         :        :- LocalLimit (55)\n         :        :  +- Project (54)\n         :        :     +- SortMergeJoin Inner (53)\n         :        :        :- Sort (49)\n         :        :        :  +- Exchange (48)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (52)\n         :        :           +- HashAggregate (51)\n         :        :              +- Exchange (50)\n         :        :                 +- HashAggregate (7)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         :        +- Sort (61)\n         :           +- HashAggregate (60)\n         :              +- Exchange (59)\n         :                 +- HashAggregate (58)\n         :                    +- HashAggregate (57)\n         :                       +- Exchange (56)\n         :                          +- HashAggregate (16)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (15)\n         +- Sort (77)\n            +- HashAggregate (76)\n               +- Exchange (75)\n                  +- HashAggregate (74)\n                     +- HashAggregate (73)\n                        +- Exchange (72)\n                           +- HashAggregate (71)\n                              +- Project (70)\n                                 +- SortMergeJoin Inner (69)\n                                    :- Sort (66)\n                                    :  +- Exchange (65)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (27)\n                                    +- Sort (68)\n                                       +- Exchange (67)\n                                          +- Project (32)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (31)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(7) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(8) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(9) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(10) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(12) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(13) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(14) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(15) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(16) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(17) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(18) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(19) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(20) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(21) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=190]\n\n(22) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(23) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(24) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(25) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(26) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(27) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(28) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(29) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(30) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(31) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(32) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(33) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(34) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(35) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(37) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(38) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(39) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=239]\n\n(40) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(41) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(42) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=242]\n\n(43) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(44) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(45) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(46) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(47) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(48) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(49) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(50) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(51) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(52) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(54) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(55) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(56) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(57) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(58) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(59) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(60) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(61) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(62) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(63) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(64) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(65) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(66) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(67) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(68) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(69) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(70) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(71) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(72) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(73) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(74) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(75) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(76) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(77) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(78) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(79) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(80) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(81) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#114], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#51], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#43], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":314,"metricType":"timing"},{"name":"peak memory","accumulatorId":315,"metricType":"size"},{"name":"spill size","accumulatorId":316,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#43 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":345,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":346,"metricType":"timing"},{"name":"peak memory","accumulatorId":344,"metricType":"size"},{"name":"number of output rows","accumulatorId":343,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":348,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":347,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":340,"metricType":"timing"},{"name":"peak memory","accumulatorId":341,"metricType":"size"},{"name":"spill size","accumulatorId":342,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":312,"metricType":"sum"},{"name":"spill size","accumulatorId":313,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#51 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=190]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":411,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":412,"metricType":"timing"},{"name":"peak memory","accumulatorId":410,"metricType":"size"},{"name":"number of output rows","accumulatorId":409,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":414,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":413,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":405,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":406,"metricType":"timing"},{"name":"peak memory","accumulatorId":404,"metricType":"size"},{"name":"number of output rows","accumulatorId":403,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":408,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":407,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":401,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":395,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":402,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":396,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":393,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":390,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":399,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":392,"metricType":"sum"},{"name":"records read","accumulatorId":389,"metricType":"sum"},{"name":"local bytes read","accumulatorId":387,"metricType":"size"},{"name":"fetch wait time","accumulatorId":388,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":385,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":391,"metricType":"sum"},{"name":"local blocks read","accumulatorId":384,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":394,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":383,"metricType":"sum"},{"name":"data size","accumulatorId":381,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":397,"metricType":"size"},{"name":"number of partitions","accumulatorId":382,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":398,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":386,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":400,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":377,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":378,"metricType":"timing"},{"name":"peak memory","accumulatorId":376,"metricType":"size"},{"name":"number of output rows","accumulatorId":375,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":380,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":379,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":372,"metricType":"timing"},{"name":"peak memory","accumulatorId":373,"metricType":"size"},{"name":"spill size","accumulatorId":374,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":310,"metricType":"sum"},{"name":"spill size","accumulatorId":311,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#114 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=242]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=239]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":511,"metricType":"timing"},{"name":"peak memory","accumulatorId":512,"metricType":"size"},{"name":"spill size","accumulatorId":513,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":537,"metricType":"timing"},{"name":"peak memory","accumulatorId":538,"metricType":"size"},{"name":"spill size","accumulatorId":539,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":509,"metricType":"sum"},{"name":"spill size","accumulatorId":510,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":505,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":506,"metricType":"timing"},{"name":"peak memory","accumulatorId":504,"metricType":"size"},{"name":"number of output rows","accumulatorId":503,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":508,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":507,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":501,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":495,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":502,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":496,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":493,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":490,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":499,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":492,"metricType":"sum"},{"name":"records read","accumulatorId":489,"metricType":"sum"},{"name":"local bytes read","accumulatorId":487,"metricType":"size"},{"name":"fetch wait time","accumulatorId":488,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":485,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":491,"metricType":"sum"},{"name":"local blocks read","accumulatorId":484,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":494,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":483,"metricType":"sum"},{"name":"data size","accumulatorId":481,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":497,"metricType":"size"},{"name":"number of partitions","accumulatorId":482,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":498,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":486,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":500,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":477,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":478,"metricType":"timing"},{"name":"peak memory","accumulatorId":476,"metricType":"size"},{"name":"number of output rows","accumulatorId":475,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":480,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":479,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":471,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":472,"metricType":"timing"},{"name":"peak memory","accumulatorId":470,"metricType":"size"},{"name":"number of output rows","accumulatorId":469,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":474,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":473,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":467,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":461,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":468,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":462,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":459,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":456,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":465,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":458,"metricType":"sum"},{"name":"records read","accumulatorId":455,"metricType":"sum"},{"name":"local bytes read","accumulatorId":453,"metricType":"size"},{"name":"fetch wait time","accumulatorId":454,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":451,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":457,"metricType":"sum"},{"name":"local blocks read","accumulatorId":450,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":460,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":449,"metricType":"sum"},{"name":"data size","accumulatorId":447,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":463,"metricType":"size"},{"name":"number of partitions","accumulatorId":448,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":452,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":466,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":443,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":444,"metricType":"timing"},{"name":"peak memory","accumulatorId":442,"metricType":"size"},{"name":"number of output rows","accumulatorId":441,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":446,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":445,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":438,"metricType":"timing"},{"name":"peak memory","accumulatorId":439,"metricType":"size"},{"name":"spill size","accumulatorId":440,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":308,"metricType":"sum"},{"name":"spill size","accumulatorId":309,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":306,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":300,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":307,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":301,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":298,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":295,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":304,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":297,"metricType":"sum"},{"name":"records read","accumulatorId":294,"metricType":"sum"},{"name":"local bytes read","accumulatorId":292,"metricType":"size"},{"name":"fetch wait time","accumulatorId":293,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":290,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":296,"metricType":"sum"},{"name":"local blocks read","accumulatorId":289,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":299,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":288,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":302,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":303,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":291,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":305,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[318,200]]}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1784293426125,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426134,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[350,200]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[416,200]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[515,200]]}
{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1784293426269,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426270,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[541,200]]}
{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1784293426291,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426293,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1784293426307,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[3],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426308,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1784293426316,"Stage Infos":[{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[4],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426318,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426272,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426293,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426306,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426317,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426326,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426317,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293427389,"Failed":false,"Killed":false,"Accumulables":[{"ID":261,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":514,"Name":"data size","Update":"1080","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":533,"Name":"shuffle bytes written","Update":"661","Value":"661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":534,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":535,"Name":"shuffle write time","Update":"27695824","Value":"27695824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":536,"Name":"duration","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":668,"Name":"internal.metrics.executorDeserializeTime","Update":320,"Value":320,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorDeserializeCpuTime","Update":109562121,"Value":109562121,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorRunTime","Update":717,"Value":717,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.executorCpuTime","Update":200276638,"Value":200276638,"Internal":true,"Count Failed Values":true},{"ID":672,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":673,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":674,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":696,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":661,"Value":661,"Internal":true,"Count Failed Values":true},{"ID":697,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":698,"Name":"internal.metrics.shuffle.write.writeTime","Update":27695824,"Value":27695824,"Internal":true,"Count Failed Values":true},{"ID":700,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":320,"Executor Deserialize CPU Time":109562121,"Executor Run Time":717,"Executor CPU Time":200276638,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":22,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":661,"Shuffle Write Time":27695824,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426308,"Completion Time":1784293427398,"Accumulables":[{"ID":261,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":514,"Name":"data size","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":533,"Name":"shuffle bytes written","Value":"661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":534,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":535,"Name":"shuffle write time","Value":"27695824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":536,"Name":"duration","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":668,"Name":"internal.metrics.executorDeserializeTime","Value":320,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorDeserializeCpuTime","Value":109562121,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorRunTime","Value":717,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.executorCpuTime","Value":200276638,"Internal":true,"Count Failed Values":true},{"ID":672,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":673,"Name":"internal.metrics.jvmGCTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":674,"Name":"internal.metrics.resultSerializationTime","Value":3,"Internal":true,"Count Failed Values":true},{"ID":696,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":661,"Internal":true,"Count Failed Values":true},{"ID":697,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":698,"Name":"internal.metrics.shuffle.write.writeTime","Value":27695824,"Internal":true,"Count Failed Values":true},{"ID":700,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1784293427404,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426272,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293427392,"Failed":false,"Killed":false,"Accumulables":[{"ID":52,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"data size","Update":"640","Value":"640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":336,"Name":"shuffle bytes written","Update":"1327","Value":"1327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":337,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"shuffle write time","Update":"35578350","Value":"35578350","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":339,"Name":"duration","Update":"128","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":563,"Name":"internal.metrics.executorDeserializeTime","Update":324,"Value":324,"Internal":true,"Count Failed Values":true},{"ID":564,"Name":"internal.metrics.executorDeserializeCpuTime","Update":126029862,"Value":126029862,"Internal":true,"Count Failed Values":true},{"ID":565,"Name":"internal.metrics.executorRunTime","Update":717,"Value":717,"Internal":true,"Count Failed Values":true},{"ID":566,"Name":"internal.metrics.executorCpuTime","Update":178816777,"Value":178816777,"Internal":true,"Count Failed Values":true},{"ID":567,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":568,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":569,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1327,"Value":1327,"Internal":true,"Count Failed Values":true},{"ID":592,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.write.writeTime","Update":35578350,"Value":35578350,"Internal":true,"Count Failed Values":true},{"ID":595,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":324,"Executor Deserialize CPU Time":126029862,"Executor Run Time":717,"Executor CPU Time":178816777,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":22,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1327,"Shuffle Write Time":35578350,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426134,"Completion Time":1784293427407,"Accumulables":[{"ID":52,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":317,"Name":"data size","Value":"640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":336,"Name":"shuffle bytes written","Value":"1327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":337,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"shuffle write time","Value":"35578350","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":339,"Name":"duration","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":563,"Name":"internal.metrics.executorDeserializeTime","Value":324,"Internal":true,"Count Failed Values":true},{"ID":564,"Name":"internal.metrics.executorDeserializeCpuTime","Value":126029862,"Internal":true,"Count Failed Values":true},{"ID":565,"Name":"internal.metrics.executorRunTime","Value":717,"Internal":true,"Count Failed Values":true},{"ID":566,"Name":"internal.metrics.executorCpuTime","Value":178816777,"Internal":true,"Count Failed Values":true},{"ID":567,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":568,"Name":"internal.metrics.jvmGCTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":569,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1327,"Internal":true,"Count Failed Values":true},{"ID":592,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":593,"Name":"internal.metrics.shuffle.write.writeTime","Value":35578350,"Internal":true,"Count Failed Values":true},{"ID":595,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1784293427409,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426293,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293427392,"Failed":false,"Killed":false,"Accumulables":[{"ID":84,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":85,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":87,"Name":"time in aggregation build","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":90,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"data size","Update":"480","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"shuffle bytes written","Update":"1157","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"shuffle write time","Update":"6373172","Value":"6373172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":371,"Name":"duration","Update":"150","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":598,"Name":"internal.metrics.executorDeserializeTime","Update":328,"Value":328,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.executorDeserializeCpuTime","Update":120719334,"Value":120719334,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.executorRunTime","Update":735,"Value":735,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.executorCpuTime","Update":250361648,"Value":250361648,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.resultSize","Update":2485,"Value":2485,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":626,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1157,"Value":1157,"Internal":true,"Count Failed Values":true},{"ID":627,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":628,"Name":"internal.metrics.shuffle.write.writeTime","Update":6373172,"Value":6373172,"Internal":true,"Count Failed Values":true},{"ID":630,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":328,"Executor Deserialize CPU Time":120719334,"Executor Run Time":735,"Executor CPU Time":250361648,"Peak Execution Memory":262144,"Result Size":2485,"JVM GC Time":22,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1157,"Shuffle Write Time":6373172,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426270,"Completion Time":1784293427410,"Accumulables":[{"ID":84,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":85,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":87,"Name":"time in aggregation build","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":90,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":349,"Name":"data size","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":368,"Name":"shuffle bytes written","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":369,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":370,"Name":"shuffle write time","Value":"6373172","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":371,"Name":"duration","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":598,"Name":"internal.metrics.executorDeserializeTime","Value":328,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.executorDeserializeCpuTime","Value":120719334,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.executorRunTime","Value":735,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.executorCpuTime","Value":250361648,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.resultSize","Value":2485,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.jvmGCTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":626,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1157,"Internal":true,"Count Failed Values":true},{"ID":627,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":628,"Name":"internal.metrics.shuffle.write.writeTime","Value":6373172,"Internal":true,"Count Failed Values":true},{"ID":630,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1784293427411,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426326,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293427392,"Failed":false,"Killed":false,"Accumulables":[{"ID":287,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":540,"Name":"data size","Update":"432","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":559,"Name":"shuffle bytes written","Update":"445","Value":"445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":560,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":561,"Name":"shuffle write time","Update":"22261281","Value":"22261281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":562,"Name":"duration","Update":"132","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":703,"Name":"internal.metrics.executorDeserializeTime","Update":308,"Value":308,"Internal":true,"Count Failed Values":true},{"ID":704,"Name":"internal.metrics.executorDeserializeCpuTime","Update":140512424,"Value":140512424,"Internal":true,"Count Failed Values":true},{"ID":705,"Name":"internal.metrics.executorRunTime","Update":717,"Value":717,"Internal":true,"Count Failed Values":true},{"ID":706,"Name":"internal.metrics.executorCpuTime","Update":198901308,"Value":198901308,"Internal":true,"Count Failed Values":true},{"ID":707,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":708,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":709,"Name":"internal.metrics.resultSerializationTime","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":731,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":445,"Value":445,"Internal":true,"Count Failed Values":true},{"ID":732,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":733,"Name":"internal.metrics.shuffle.write.writeTime","Update":22261281,"Value":22261281,"Internal":true,"Count Failed Values":true},{"ID":735,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":308,"Executor Deserialize CPU Time":140512424,"Executor Run Time":717,"Executor CPU Time":198901308,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":22,"Result Serialization Time":5,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":445,"Shuffle Write Time":22261281,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426318,"Completion Time":1784293427412,"Accumulables":[{"ID":287,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":540,"Name":"data size","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":559,"Name":"shuffle bytes written","Value":"445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":560,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":561,"Name":"shuffle write time","Value":"22261281","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":562,"Name":"duration","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":703,"Name":"internal.metrics.executorDeserializeTime","Value":308,"Internal":true,"Count Failed Values":true},{"ID":704,"Name":"internal.metrics.executorDeserializeCpuTime","Value":140512424,"Internal":true,"Count Failed Values":true},{"ID":705,"Name":"internal.metrics.executorRunTime","Value":717,"Internal":true,"Count Failed Values":true},{"ID":706,"Name":"internal.metrics.executorCpuTime","Value":198901308,"Internal":true,"Count Failed Values":true},{"ID":707,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":708,"Name":"internal.metrics.jvmGCTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":709,"Name":"internal.metrics.resultSerializationTime","Value":5,"Internal":true,"Count Failed Values":true},{"ID":731,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":445,"Internal":true,"Count Failed Values":true},{"ID":732,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":733,"Name":"internal.metrics.shuffle.write.writeTime","Value":22261281,"Internal":true,"Count Failed Values":true},{"ID":735,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1784293427413,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (86)\n+- == Current Plan ==\n   CollectLimit (52)\n   +- Project (51)\n      +- BroadcastHashJoin LeftOuter BuildRight (50)\n         :- LocalLimit (28)\n         :  +- Project (27)\n         :     +- SortMergeJoin LeftOuter (26)\n         :        :- Sort (16)\n         :        :  +- LocalLimit (15)\n         :        :     +- Project (14)\n         :        :        +- BroadcastHashJoin Inner BuildRight (13)\n         :        :           :- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :           :  +- Exchange (3)\n         :        :           :     +- * Project (2)\n         :        :           :        +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :           +- BroadcastQueryStage (12)\n         :        :              +- BroadcastExchange (11)\n         :        :                 +- * HashAggregate (10)\n         :        :                    +- AQEShuffleRead (9)\n         :        :                       +- ShuffleQueryStage (8), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                          +- Exchange (7)\n         :        :                             +- * HashAggregate (6)\n         :        :                                +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (25)\n         :           +- HashAggregate (24)\n         :              +- Exchange (23)\n         :                 +- HashAggregate (22)\n         :                    +- HashAggregate (21)\n         :                       +- ShuffleQueryStage (20)\n         :                          +- Exchange (19)\n         :                             +- * HashAggregate (18)\n         :                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (17)\n         +- BroadcastExchange (49)\n            +- HashAggregate (48)\n               +- Exchange (47)\n                  +- HashAggregate (46)\n                     +- HashAggregate (45)\n                        +- ShuffleQueryStage (44)\n                           +- Exchange (43)\n                              +- * HashAggregate (42)\n                                 +- * Project (41)\n                                    +- * SortMergeJoin Inner (40)\n                                       :- * Sort (33)\n                                       :  +- AQEShuffleRead (32)\n                                       :     +- ShuffleQueryStage (31), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :        +- Exchange (30)\n                                       :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (29)\n                                       +- * Sort (39)\n                                          +- AQEShuffleRead (38)\n                                             +- ShuffleQueryStage (37), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                +- Exchange (36)\n                                                   +- * Project (35)\n                                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n+- == Initial Plan ==\n   CollectLimit (85)\n   +- Project (84)\n      +- SortMergeJoin LeftOuter (83)\n         :- LocalLimit (69)\n         :  +- Project (68)\n         :     +- SortMergeJoin LeftOuter (67)\n         :        :- LocalLimit (60)\n         :        :  +- Project (59)\n         :        :     +- SortMergeJoin Inner (58)\n         :        :        :- Sort (54)\n         :        :        :  +- Exchange (53)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (57)\n         :        :           +- HashAggregate (56)\n         :        :              +- Exchange (55)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (66)\n         :           +- HashAggregate (65)\n         :              +- Exchange (64)\n         :                 +- HashAggregate (63)\n         :                    +- HashAggregate (62)\n         :                       +- Exchange (61)\n         :                          +- HashAggregate (18)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (17)\n         +- Sort (82)\n            +- HashAggregate (81)\n               +- Exchange (80)\n                  +- HashAggregate (79)\n                     +- HashAggregate (78)\n                        +- Exchange (77)\n                           +- HashAggregate (76)\n                              +- Project (75)\n                                 +- SortMergeJoin Inner (74)\n                                    :- Sort (71)\n                                    :  +- Exchange (70)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (29)\n                                    +- Sort (73)\n                                       +- Exchange (72)\n                                          +- Project (35)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(12) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(13) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(15) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(16) Sort\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(17) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(18) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(19) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(20) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(21) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(22) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(23) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=190]\n\n(24) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(25) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(26) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(28) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(29) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(30) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(31) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(32) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(33) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(34) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(35) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(36) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(37) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(38) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(39) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(40) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(41) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(43) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(44) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(45) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(46) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(47) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]\n\n(48) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(49) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=524]\n\n(50) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(51) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(52) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(53) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(54) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(55) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(56) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(57) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(58) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(59) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(60) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(61) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(62) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(63) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(64) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(65) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(66) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(69) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(70) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(71) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(73) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(74) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(75) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(76) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(77) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(78) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(79) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(80) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(81) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(82) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(83) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(84) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(85) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(86) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#10], [user_id#51], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":764,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":761,"metricType":"timing"},{"name":"peak memory","accumulatorId":762,"metricType":"size"},{"name":"spill size","accumulatorId":763,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#51 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=190]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":411,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":412,"metricType":"timing"},{"name":"peak memory","accumulatorId":410,"metricType":"size"},{"name":"number of output rows","accumulatorId":409,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":414,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":413,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":405,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":406,"metricType":"timing"},{"name":"peak memory","accumulatorId":404,"metricType":"size"},{"name":"number of output rows","accumulatorId":403,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":408,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":407,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":401,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":395,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":402,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":396,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":393,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":390,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":399,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":392,"metricType":"sum"},{"name":"records read","accumulatorId":389,"metricType":"sum"},{"name":"local bytes read","accumulatorId":387,"metricType":"size"},{"name":"fetch wait time","accumulatorId":388,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":385,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":391,"metricType":"sum"},{"name":"local blocks read","accumulatorId":384,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":394,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":383,"metricType":"sum"},{"name":"data size","accumulatorId":381,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":397,"metricType":"size"},{"name":"number of partitions","accumulatorId":382,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":398,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":386,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":400,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":377,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":378,"metricType":"timing"},{"name":"peak memory","accumulatorId":376,"metricType":"size"},{"name":"number of output rows","accumulatorId":375,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":380,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":379,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":780,"metricType":"timing"},{"name":"peak memory","accumulatorId":781,"metricType":"size"},{"name":"spill size","accumulatorId":782,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":759,"metricType":"sum"},{"name":"spill size","accumulatorId":760,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=524]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":818,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":819,"metricType":"timing"},{"name":"peak memory","accumulatorId":817,"metricType":"size"},{"name":"number of output rows","accumulatorId":816,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":821,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":820,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":814,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":808,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":815,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":809,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":806,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":803,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":812,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":805,"metricType":"sum"},{"name":"records read","accumulatorId":802,"metricType":"sum"},{"name":"local bytes read","accumulatorId":800,"metricType":"size"},{"name":"fetch wait time","accumulatorId":801,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":798,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":804,"metricType":"sum"},{"name":"local blocks read","accumulatorId":797,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":807,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":796,"metricType":"sum"},{"name":"data size","accumulatorId":794,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":810,"metricType":"size"},{"name":"number of partitions","accumulatorId":795,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":811,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":799,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":813,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":790,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":791,"metricType":"timing"},{"name":"peak memory","accumulatorId":789,"metricType":"size"},{"name":"number of output rows","accumulatorId":788,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":793,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":792,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":787,"metricType":"timing"},{"name":"time to build","accumulatorId":786,"metricType":"timing"},{"name":"time to collect","accumulatorId":785,"metricType":"timing"},{"name":"number of output rows","accumulatorId":784,"metricType":"sum"},{"name":"data size","accumulatorId":783,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":758,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":756,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":750,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":757,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":751,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":748,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":745,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":754,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":747,"metricType":"sum"},{"name":"records read","accumulatorId":744,"metricType":"sum"},{"name":"local bytes read","accumulatorId":742,"metricType":"size"},{"name":"fetch wait time","accumulatorId":743,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":740,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":746,"metricType":"sum"},{"name":"local blocks read","accumulatorId":739,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":749,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":738,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":752,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":753,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":741,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":755,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[777,1],[779,1],[778,1210]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[863,1],[865,1],[864,689]]}
{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1784293427650,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"5\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"28\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[5],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5,6],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"19\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.job.tags":"broadcast exchange (runId 49e404e8-31ae-48d3-aed0-e702a05ed2fe)","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"28\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[5],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427661,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"19\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.job.tags":"broadcast exchange (runId 49e404e8-31ae-48d3-aed0-e702a05ed2fe)","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[870,1],[872,1],[871,467]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[829,200]]}
{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293426306,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293427664,"Failed":false,"Killed":false,"Accumulables":[{"ID":156,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":159,"Name":"time in aggregation build","Update":"284","Value":"284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":415,"Name":"data size","Update":"2160","Value":"2160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":434,"Name":"shuffle bytes written","Update":"3641","Value":"3641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":435,"Name":"shuffle records written","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":436,"Name":"shuffle write time","Update":"5814135","Value":"5814135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":437,"Name":"duration","Update":"424","Value":"424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":633,"Name":"internal.metrics.executorDeserializeTime","Update":329,"Value":329,"Internal":true,"Count Failed Values":true},{"ID":634,"Name":"internal.metrics.executorDeserializeCpuTime","Update":129133808,"Value":129133808,"Internal":true,"Count Failed Values":true},{"ID":635,"Name":"internal.metrics.executorRunTime","Update":1010,"Value":1010,"Internal":true,"Count Failed Values":true},{"ID":636,"Name":"internal.metrics.executorCpuTime","Update":472662058,"Value":472662058,"Internal":true,"Count Failed Values":true},{"ID":637,"Name":"internal.metrics.resultSize","Update":2485,"Value":2485,"Internal":true,"Count Failed Values":true},{"ID":638,"Name":"internal.metrics.jvmGCTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":642,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":661,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":3641,"Value":3641,"Internal":true,"Count Failed Values":true},{"ID":662,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":663,"Name":"internal.metrics.shuffle.write.writeTime","Update":5814135,"Value":5814135,"Internal":true,"Count Failed Values":true},{"ID":665,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":329,"Executor Deserialize CPU Time":129133808,"Executor Run Time":1010,"Executor CPU Time":472662058,"Peak Execution Memory":262144,"Result Size":2485,"JVM GC Time":22,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":3641,"Shuffle Write Time":5814135,"Shuffle Records Written":54},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293426293,"Completion Time":1784293427766,"Accumulables":[{"ID":156,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":157,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":159,"Name":"time in aggregation build","Value":"284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":162,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":415,"Name":"data size","Value":"2160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":434,"Name":"shuffle bytes written","Value":"3641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":435,"Name":"shuffle records written","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":436,"Name":"shuffle write time","Value":"5814135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":437,"Name":"duration","Value":"424","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":633,"Name":"internal.metrics.executorDeserializeTime","Value":329,"Internal":true,"Count Failed Values":true},{"ID":634,"Name":"internal.metrics.executorDeserializeCpuTime","Value":129133808,"Internal":true,"Count Failed Values":true},{"ID":635,"Name":"internal.metrics.executorRunTime","Value":1010,"Internal":true,"Count Failed Values":true},{"ID":636,"Name":"internal.metrics.executorCpuTime","Value":472662058,"Internal":true,"Count Failed Values":true},{"ID":637,"Name":"internal.metrics.resultSize","Value":2485,"Internal":true,"Count Failed Values":true},{"ID":638,"Name":"internal.metrics.jvmGCTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":642,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":661,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":3641,"Internal":true,"Count Failed Values":true},{"ID":662,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":54,"Internal":true,"Count Failed Values":true},{"ID":663,"Name":"internal.metrics.shuffle.write.writeTime","Value":5814135,"Internal":true,"Count Failed Values":true},{"ID":665,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1784293427767,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1784293427769,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19,21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"38\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7,8],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[9,7,8],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19,21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"38\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7,8],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427772,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":6,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427765,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":9,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427789,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (87)\n+- == Current Plan ==\n   CollectLimit (53)\n   +- Project (52)\n      +- BroadcastHashJoin LeftOuter BuildRight (51)\n         :- LocalLimit (29)\n         :  +- Project (28)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (27)\n         :        :- LocalLimit (15)\n         :        :  +- Project (14)\n         :        :     +- BroadcastHashJoin Inner BuildRight (13)\n         :        :        :- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :        :  +- Exchange (3)\n         :        :        :     +- * Project (2)\n         :        :        :        +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- BroadcastQueryStage (12)\n         :        :           +- BroadcastExchange (11)\n         :        :              +- * HashAggregate (10)\n         :        :                 +- AQEShuffleRead (9)\n         :        :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                       +- Exchange (7)\n         :        :                          +- * HashAggregate (6)\n         :        :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- BroadcastExchange (26)\n         :           +- HashAggregate (25)\n         :              +- ShuffleQueryStage (24)\n         :                 +- Exchange (23)\n         :                    +- * HashAggregate (22)\n         :                       +- * HashAggregate (21)\n         :                          +- AQEShuffleRead (20)\n         :                             +- ShuffleQueryStage (19), Statistics(sizeInBytes=2.1 KiB, rowCount=54)\n         :                                +- Exchange (18)\n         :                                   +- * HashAggregate (17)\n         :                                      +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- BroadcastExchange (50)\n            +- HashAggregate (49)\n               +- Exchange (48)\n                  +- HashAggregate (47)\n                     +- HashAggregate (46)\n                        +- ShuffleQueryStage (45)\n                           +- Exchange (44)\n                              +- * HashAggregate (43)\n                                 +- * Project (42)\n                                    +- * SortMergeJoin Inner (41)\n                                       :- * Sort (34)\n                                       :  +- AQEShuffleRead (33)\n                                       :     +- ShuffleQueryStage (32), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :        +- Exchange (31)\n                                       :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (30)\n                                       +- * Sort (40)\n                                          +- AQEShuffleRead (39)\n                                             +- ShuffleQueryStage (38), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                +- Exchange (37)\n                                                   +- * Project (36)\n                                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   CollectLimit (86)\n   +- Project (85)\n      +- SortMergeJoin LeftOuter (84)\n         :- LocalLimit (70)\n         :  +- Project (69)\n         :     +- SortMergeJoin LeftOuter (68)\n         :        :- LocalLimit (61)\n         :        :  +- Project (60)\n         :        :     +- SortMergeJoin Inner (59)\n         :        :        :- Sort (55)\n         :        :        :  +- Exchange (54)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (58)\n         :        :           +- HashAggregate (57)\n         :        :              +- Exchange (56)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (67)\n         :           +- HashAggregate (66)\n         :              +- Exchange (65)\n         :                 +- HashAggregate (64)\n         :                    +- HashAggregate (63)\n         :                       +- Exchange (62)\n         :                          +- HashAggregate (17)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- Sort (83)\n            +- HashAggregate (82)\n               +- Exchange (81)\n                  +- HashAggregate (80)\n                     +- HashAggregate (79)\n                        +- Exchange (78)\n                           +- HashAggregate (77)\n                              +- Project (76)\n                                 +- SortMergeJoin Inner (75)\n                                    :- Sort (72)\n                                    :  +- Exchange (71)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (30)\n                                    +- Sort (74)\n                                       +- Exchange (73)\n                                          +- Project (36)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(12) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(13) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(15) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(16) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(17) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(18) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(19) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(20) AQEShuffleRead\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: coalesced\n\n(21) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(22) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(23) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]\n\n(24) ShuffleQueryStage\nOutput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: 7\n\n(25) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(26) BroadcastExchange\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=593]\n\n(27) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(29) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(30) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(31) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(32) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(33) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(34) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(35) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(36) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(37) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(38) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(40) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(42) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(43) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(44) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(45) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(46) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(47) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(48) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]\n\n(49) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(50) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=566]\n\n(51) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(52) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(53) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(54) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(55) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(56) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(57) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(58) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(60) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(61) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(62) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(63) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(64) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(65) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(66) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(67) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(68) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(69) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(70) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(71) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(72) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(73) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(74) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(76) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(77) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(78) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(79) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(80) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(81) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(82) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(83) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(84) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(85) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(86) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(87) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#51], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":965,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=593]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1012,"metricType":"sum"},{"name":"partition data size","accumulatorId":1013,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1014,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1008,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1009,"metricType":"timing"},{"name":"peak memory","accumulatorId":1007,"metricType":"size"},{"name":"number of output rows","accumulatorId":1006,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1010,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1002,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1003,"metricType":"timing"},{"name":"peak memory","accumulatorId":1001,"metricType":"size"},{"name":"number of output rows","accumulatorId":1000,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1005,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1004,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":988,"metricType":"sum"},{"name":"records read","accumulatorId":985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":979,"metricType":"sum"},{"name":"data size","accumulatorId":977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":993,"metricType":"size"},{"name":"number of partitions","accumulatorId":978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":973,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":974,"metricType":"timing"},{"name":"peak memory","accumulatorId":972,"metricType":"size"},{"name":"number of output rows","accumulatorId":971,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":976,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":975,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":970,"metricType":"timing"},{"name":"time to build","accumulatorId":969,"metricType":"timing"},{"name":"time to collect","accumulatorId":968,"metricType":"timing"},{"name":"number of output rows","accumulatorId":967,"metricType":"sum"},{"name":"data size","accumulatorId":966,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":964,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=566]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":818,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":819,"metricType":"timing"},{"name":"peak memory","accumulatorId":817,"metricType":"size"},{"name":"number of output rows","accumulatorId":816,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":821,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":820,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":814,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":808,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":815,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":809,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":806,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":803,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":812,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":805,"metricType":"sum"},{"name":"records read","accumulatorId":802,"metricType":"sum"},{"name":"local bytes read","accumulatorId":800,"metricType":"size"},{"name":"fetch wait time","accumulatorId":801,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":798,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":804,"metricType":"sum"},{"name":"local blocks read","accumulatorId":797,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":807,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":796,"metricType":"sum"},{"name":"data size","accumulatorId":794,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":810,"metricType":"size"},{"name":"number of partitions","accumulatorId":795,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":811,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":799,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":813,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":790,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":791,"metricType":"timing"},{"name":"peak memory","accumulatorId":789,"metricType":"size"},{"name":"number of output rows","accumulatorId":788,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":793,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":792,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1019,"metricType":"timing"},{"name":"time to build","accumulatorId":1018,"metricType":"timing"},{"name":"time to collect","accumulatorId":1017,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1016,"metricType":"sum"},{"name":"data size","accumulatorId":1015,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":963,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":961,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":955,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":962,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":956,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":953,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":950,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":959,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":952,"metricType":"sum"},{"name":"records read","accumulatorId":949,"metricType":"sum"},{"name":"local bytes read","accumulatorId":947,"metricType":"size"},{"name":"fetch wait time","accumulatorId":948,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":945,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":951,"metricType":"sum"},{"name":"local blocks read","accumulatorId":944,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":954,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":943,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":957,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":958,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":946,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":960,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1012,1],[1014,1],[1013,3820]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[978,200]]}
{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1784293427898,"Stage Infos":[{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[10,11],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427908,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427924,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":6,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427765,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428072,"Failed":false,"Killed":false,"Accumulables":[{"ID":352,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":355,"Name":"local bytes read","Update":"1157","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":356,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":357,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":770,"Name":"duration","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":771,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":772,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":774,"Name":"time in aggregation build","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":775,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":873,"Name":"internal.metrics.executorDeserializeTime","Update":219,"Value":219,"Internal":true,"Count Failed Values":true},{"ID":874,"Name":"internal.metrics.executorDeserializeCpuTime","Update":94439761,"Value":94439761,"Internal":true,"Count Failed Values":true},{"ID":875,"Name":"internal.metrics.executorRunTime","Update":70,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":876,"Name":"internal.metrics.executorCpuTime","Update":40408691,"Value":40408691,"Internal":true,"Count Failed Values":true},{"ID":877,"Name":"internal.metrics.resultSize","Update":5098,"Value":5098,"Internal":true,"Count Failed Values":true},{"ID":879,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":882,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":884,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":885,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":886,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":887,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":888,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1157,"Value":1157,"Internal":true,"Count Failed Values":true},{"ID":889,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":890,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":891,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":892,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":893,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":894,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":895,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":896,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":897,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":898,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":899,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":900,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":219,"Executor Deserialize CPU Time":94439761,"Executor Run Time":70,"Executor CPU Time":40408691,"Peak Execution Memory":4456432,"Result Size":5098,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1157,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"28\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"22\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[5],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427661,"Completion Time":1784293428079,"Accumulables":[{"ID":352,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":355,"Name":"local bytes read","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":356,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":357,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":770,"Name":"duration","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":771,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":772,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":774,"Name":"time in aggregation build","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":775,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":873,"Name":"internal.metrics.executorDeserializeTime","Value":219,"Internal":true,"Count Failed Values":true},{"ID":874,"Name":"internal.metrics.executorDeserializeCpuTime","Value":94439761,"Internal":true,"Count Failed Values":true},{"ID":875,"Name":"internal.metrics.executorRunTime","Value":70,"Internal":true,"Count Failed Values":true},{"ID":876,"Name":"internal.metrics.executorCpuTime","Value":40408691,"Internal":true,"Count Failed Values":true},{"ID":877,"Name":"internal.metrics.resultSize","Value":5098,"Internal":true,"Count Failed Values":true},{"ID":879,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":882,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":884,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":885,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":886,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":887,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":888,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1157,"Internal":true,"Count Failed Values":true},{"ID":889,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":890,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":891,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":892,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":893,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":894,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":895,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":896,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":897,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":898,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":899,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":900,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1784293428087,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[769,17],[768,29],[767,517],[766,20],[765,1048736]]}
{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427924,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428142,"Failed":false,"Killed":false,"Accumulables":[{"ID":418,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":421,"Name":"local bytes read","Update":"3641","Value":"3641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":422,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":423,"Name":"records read","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":977,"Name":"data size","Update":"440","Value":"440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"shuffle bytes written","Update":"880","Value":"880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":998,"Name":"shuffle write time","Update":"5387111","Value":"5387111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"duration","Update":"50","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1001,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1003,"Name":"time in aggregation build","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1020,"Name":"internal.metrics.executorDeserializeTime","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":1021,"Name":"internal.metrics.executorDeserializeCpuTime","Update":17562626,"Value":17562626,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorRunTime","Update":140,"Value":140,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorCpuTime","Update":112531773,"Value":112531773,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1029,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1031,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1032,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1033,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1034,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1035,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3641,"Value":3641,"Internal":true,"Count Failed Values":true},{"ID":1036,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1037,"Name":"internal.metrics.shuffle.read.recordsRead","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":880,"Value":880,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1050,"Name":"internal.metrics.shuffle.write.writeTime","Update":5387111,"Value":5387111,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":61,"Executor Deserialize CPU Time":17562626,"Executor Run Time":140,"Executor CPU Time":112531773,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3641,"Total Records Read":54,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":880,"Shuffle Write Time":5387111,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427908,"Completion Time":1784293428144,"Accumulables":[{"ID":418,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":421,"Name":"local bytes read","Value":"3641","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":422,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":423,"Name":"records read","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":977,"Name":"data size","Value":"440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"shuffle bytes written","Value":"880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":998,"Name":"shuffle write time","Value":"5387111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"duration","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1001,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1003,"Name":"time in aggregation build","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1020,"Name":"internal.metrics.executorDeserializeTime","Value":61,"Internal":true,"Count Failed Values":true},{"ID":1021,"Name":"internal.metrics.executorDeserializeCpuTime","Value":17562626,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.executorRunTime","Value":140,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.executorCpuTime","Value":112531773,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1029,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1031,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1032,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1033,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1034,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1035,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3641,"Internal":true,"Count Failed Values":true},{"ID":1036,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1037,"Name":"internal.metrics.shuffle.read.recordsRead","Value":54,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1041,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1043,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1044,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1045,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":880,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1050,"Name":"internal.metrics.shuffle.write.writeTime","Value":5387111,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1784293428144,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (89)\n+- == Current Plan ==\n   CollectLimit (55)\n   +- Project (54)\n      +- BroadcastHashJoin LeftOuter BuildRight (53)\n         :- LocalLimit (31)\n         :  +- Project (30)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (29)\n         :        :- LocalLimit (15)\n         :        :  +- Project (14)\n         :        :     +- BroadcastHashJoin Inner BuildRight (13)\n         :        :        :- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :        :  +- Exchange (3)\n         :        :        :     +- * Project (2)\n         :        :        :        +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- BroadcastQueryStage (12), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n         :        :           +- BroadcastExchange (11)\n         :        :              +- * HashAggregate (10)\n         :        :                 +- AQEShuffleRead (9)\n         :        :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                       +- Exchange (7)\n         :        :                          +- * HashAggregate (6)\n         :        :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- BroadcastQueryStage (28)\n         :           +- BroadcastExchange (27)\n         :              +- * HashAggregate (26)\n         :                 +- AQEShuffleRead (25)\n         :                    +- ShuffleQueryStage (24), Statistics(sizeInBytes=440.0 B, rowCount=11)\n         :                       +- Exchange (23)\n         :                          +- * HashAggregate (22)\n         :                             +- * HashAggregate (21)\n         :                                +- AQEShuffleRead (20)\n         :                                   +- ShuffleQueryStage (19), Statistics(sizeInBytes=2.1 KiB, rowCount=54)\n         :                                      +- Exchange (18)\n         :                                         +- * HashAggregate (17)\n         :                                            +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- BroadcastExchange (52)\n            +- HashAggregate (51)\n               +- Exchange (50)\n                  +- HashAggregate (49)\n                     +- HashAggregate (48)\n                        +- ShuffleQueryStage (47)\n                           +- Exchange (46)\n                              +- * HashAggregate (45)\n                                 +- * Project (44)\n                                    +- * SortMergeJoin Inner (43)\n                                       :- * Sort (36)\n                                       :  +- AQEShuffleRead (35)\n                                       :     +- ShuffleQueryStage (34), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :        +- Exchange (33)\n                                       :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                       +- * Sort (42)\n                                          +- AQEShuffleRead (41)\n                                             +- ShuffleQueryStage (40), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                +- Exchange (39)\n                                                   +- * Project (38)\n                                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (88)\n   +- Project (87)\n      +- SortMergeJoin LeftOuter (86)\n         :- LocalLimit (72)\n         :  +- Project (71)\n         :     +- SortMergeJoin LeftOuter (70)\n         :        :- LocalLimit (63)\n         :        :  +- Project (62)\n         :        :     +- SortMergeJoin Inner (61)\n         :        :        :- Sort (57)\n         :        :        :  +- Exchange (56)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (60)\n         :        :           +- HashAggregate (59)\n         :        :              +- Exchange (58)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (69)\n         :           +- HashAggregate (68)\n         :              +- Exchange (67)\n         :                 +- HashAggregate (66)\n         :                    +- HashAggregate (65)\n         :                       +- Exchange (64)\n         :                          +- HashAggregate (17)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- Sort (85)\n            +- HashAggregate (84)\n               +- Exchange (83)\n                  +- HashAggregate (82)\n                     +- HashAggregate (81)\n                        +- Exchange (80)\n                           +- HashAggregate (79)\n                              +- Project (78)\n                                 +- SortMergeJoin Inner (77)\n                                    :- Sort (74)\n                                    :  +- Exchange (73)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                    +- Sort (76)\n                                       +- Exchange (75)\n                                          +- Project (38)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(12) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(13) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(15) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(16) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(17) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(18) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(19) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(20) AQEShuffleRead\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: coalesced\n\n(21) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(22) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(23) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]\n\n(24) ShuffleQueryStage\nOutput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: 7\n\n(25) AQEShuffleRead\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 11]\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(27) BroadcastExchange\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]\n\n(28) BroadcastQueryStage\nOutput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 8\n\n(29) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(30) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(31) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(32) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(33) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(34) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(35) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(36) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(37) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(38) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(39) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(40) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(41) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(42) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(44) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(45) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(46) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(47) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(48) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(49) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(50) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]\n\n(51) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(52) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=566]\n\n(53) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(54) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(55) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(56) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(57) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(58) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(59) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(60) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(61) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(62) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(63) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(64) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(65) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(66) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(67) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(68) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(69) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(70) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(71) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(72) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(73) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(74) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(75) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(76) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(77) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(78) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(79) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(80) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(81) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(82) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(83) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(84) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(85) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(86) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(87) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(88) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(89) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#51], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":965,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1012,"metricType":"sum"},{"name":"partition data size","accumulatorId":1013,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1014,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1008,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1009,"metricType":"timing"},{"name":"peak memory","accumulatorId":1007,"metricType":"size"},{"name":"number of output rows","accumulatorId":1006,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1010,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1002,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1003,"metricType":"timing"},{"name":"peak memory","accumulatorId":1001,"metricType":"size"},{"name":"number of output rows","accumulatorId":1000,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1005,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1004,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":988,"metricType":"sum"},{"name":"records read","accumulatorId":985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":979,"metricType":"sum"},{"name":"data size","accumulatorId":977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":993,"metricType":"size"},{"name":"number of partitions","accumulatorId":978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1089,"metricType":"sum"},{"name":"partition data size","accumulatorId":1090,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1091,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1085,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1086,"metricType":"timing"},{"name":"peak memory","accumulatorId":1084,"metricType":"size"},{"name":"number of output rows","accumulatorId":1083,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1088,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1087,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1082,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1081,"metricType":"timing"},{"name":"time to build","accumulatorId":1080,"metricType":"timing"},{"name":"time to collect","accumulatorId":1079,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1078,"metricType":"sum"},{"name":"data size","accumulatorId":1077,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1076,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=566]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=522]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":818,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":819,"metricType":"timing"},{"name":"peak memory","accumulatorId":817,"metricType":"size"},{"name":"number of output rows","accumulatorId":816,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":821,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":820,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":814,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":808,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":815,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":809,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":806,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":803,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":812,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":805,"metricType":"sum"},{"name":"records read","accumulatorId":802,"metricType":"sum"},{"name":"local bytes read","accumulatorId":800,"metricType":"size"},{"name":"fetch wait time","accumulatorId":801,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":798,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":804,"metricType":"sum"},{"name":"local blocks read","accumulatorId":797,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":807,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":796,"metricType":"sum"},{"name":"data size","accumulatorId":794,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":810,"metricType":"size"},{"name":"number of partitions","accumulatorId":795,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":811,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":799,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":813,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":790,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":791,"metricType":"timing"},{"name":"peak memory","accumulatorId":789,"metricType":"size"},{"name":"number of output rows","accumulatorId":788,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":793,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":792,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1019,"metricType":"timing"},{"name":"time to build","accumulatorId":1018,"metricType":"timing"},{"name":"time to collect","accumulatorId":1017,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1016,"metricType":"sum"},{"name":"data size","accumulatorId":1015,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1075,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1073,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1067,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1074,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1068,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1065,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1062,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1071,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1064,"metricType":"sum"},{"name":"records read","accumulatorId":1061,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1059,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1060,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1057,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1063,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1056,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1066,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1055,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1069,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1070,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1058,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1072,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1089,1],[1091,1],[1090,880]]}
{"Event":"SparkListenerJobStart","Job ID":8,"Submission Time":1784293428212,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[12],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"51\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"55\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"52\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[12,13,14],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"51\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId df2f252f-452f-4e19-9e3f-c420e858e032)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"51\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"55\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"52\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428214,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"51\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId df2f252f-452f-4e19-9e3f-c420e858e032)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":14,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428226,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":9,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293427789,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428266,"Failed":false,"Killed":false,"Accumulables":[{"ID":517,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"local bytes read","Update":"661","Value":"661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":543,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":546,"Name":"local bytes read","Update":"445","Value":"445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":547,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":548,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":828,"Name":"data size","Update":"3936","Value":"3936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":847,"Name":"shuffle bytes written","Update":"7703","Value":"7703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":848,"Name":"shuffle records written","Update":"164","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":849,"Name":"shuffle write time","Update":"20910711","Value":"20910711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":850,"Name":"duration","Update":"81","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":851,"Name":"number of output rows","Update":"164","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":852,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":854,"Name":"time in aggregation build","Update":"50","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":857,"Name":"number of output rows","Update":"164","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":859,"Name":"duration","Update":"178","Value":"178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":860,"Name":"sort time","Update":"36","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":862,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":866,"Name":"duration","Update":"116","Value":"116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":867,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":868,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":869,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"internal.metrics.executorDeserializeTime","Update":106,"Value":106,"Internal":true,"Count Failed Values":true},{"ID":909,"Name":"internal.metrics.executorDeserializeCpuTime","Update":56218740,"Value":56218740,"Internal":true,"Count Failed Values":true},{"ID":910,"Name":"internal.metrics.executorRunTime","Update":358,"Value":358,"Internal":true,"Count Failed Values":true},{"ID":911,"Name":"internal.metrics.executorCpuTime","Update":208302798,"Value":208302798,"Internal":true,"Count Failed Values":true},{"ID":912,"Name":"internal.metrics.resultSize","Update":6805,"Value":6805,"Internal":true,"Count Failed Values":true},{"ID":917,"Name":"internal.metrics.peakExecutionMemory","Update":8781792,"Value":8781792,"Internal":true,"Count Failed Values":true},{"ID":919,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":920,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":921,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":922,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":923,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1106,"Value":1106,"Internal":true,"Count Failed Values":true},{"ID":924,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":925,"Name":"internal.metrics.shuffle.read.recordsRead","Update":63,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":926,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":927,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":928,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":929,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":930,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":931,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":932,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":933,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":934,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":935,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":936,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":7703,"Value":7703,"Internal":true,"Count Failed Values":true},{"ID":937,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":164,"Value":164,"Internal":true,"Count Failed Values":true},{"ID":938,"Name":"internal.metrics.shuffle.write.writeTime","Update":20910711,"Value":20910711,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":106,"Executor Deserialize CPU Time":56218740,"Executor Run Time":358,"Executor CPU Time":208302798,"Peak Execution Memory":8781792,"Result Size":6805,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1106,"Total Records Read":63,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":7703,"Shuffle Write Time":20910711,"Shuffle Records Written":164},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19,21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"38\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[7,8],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293427772,"Completion Time":1784293428268,"Accumulables":[{"ID":517,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"local bytes read","Value":"661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":543,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":546,"Name":"local bytes read","Value":"445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":547,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":548,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":828,"Name":"data size","Value":"3936","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":847,"Name":"shuffle bytes written","Value":"7703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":848,"Name":"shuffle records written","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":849,"Name":"shuffle write time","Value":"20910711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":850,"Name":"duration","Value":"81","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":851,"Name":"number of output rows","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":852,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":854,"Name":"time in aggregation build","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":857,"Name":"number of output rows","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":859,"Name":"duration","Value":"178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":860,"Name":"sort time","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":862,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":866,"Name":"duration","Value":"116","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":867,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":868,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":869,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"internal.metrics.executorDeserializeTime","Value":106,"Internal":true,"Count Failed Values":true},{"ID":909,"Name":"internal.metrics.executorDeserializeCpuTime","Value":56218740,"Internal":true,"Count Failed Values":true},{"ID":910,"Name":"internal.metrics.executorRunTime","Value":358,"Internal":true,"Count Failed Values":true},{"ID":911,"Name":"internal.metrics.executorCpuTime","Value":208302798,"Internal":true,"Count Failed Values":true},{"ID":912,"Name":"internal.metrics.resultSize","Value":6805,"Internal":true,"Count Failed Values":true},{"ID":917,"Name":"internal.metrics.peakExecutionMemory","Value":8781792,"Internal":true,"Count Failed Values":true},{"ID":919,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":920,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":921,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":922,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":923,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1106,"Internal":true,"Count Failed Values":true},{"ID":924,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":925,"Name":"internal.metrics.shuffle.read.recordsRead","Value":63,"Internal":true,"Count Failed Values":true},{"ID":926,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":927,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":928,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":929,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":930,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":931,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":932,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":933,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":934,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":935,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":936,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":7703,"Internal":true,"Count Failed Values":true},{"ID":937,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":164,"Internal":true,"Count Failed Values":true},{"ID":938,"Name":"internal.metrics.shuffle.write.writeTime","Value":20910711,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1784293428270,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (91)\n+- == Current Plan ==\n   CollectLimit (57)\n   +- Project (56)\n      +- BroadcastHashJoin LeftOuter BuildRight (55)\n         :- LocalLimit (31)\n         :  +- Project (30)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (29)\n         :        :- LocalLimit (15)\n         :        :  +- Project (14)\n         :        :     +- BroadcastHashJoin Inner BuildRight (13)\n         :        :        :- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :        :  +- Exchange (3)\n         :        :        :     +- * Project (2)\n         :        :        :        +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- BroadcastQueryStage (12), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n         :        :           +- BroadcastExchange (11)\n         :        :              +- * HashAggregate (10)\n         :        :                 +- AQEShuffleRead (9)\n         :        :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                       +- Exchange (7)\n         :        :                          +- * HashAggregate (6)\n         :        :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- BroadcastQueryStage (28)\n         :           +- BroadcastExchange (27)\n         :              +- * HashAggregate (26)\n         :                 +- AQEShuffleRead (25)\n         :                    +- ShuffleQueryStage (24), Statistics(sizeInBytes=440.0 B, rowCount=11)\n         :                       +- Exchange (23)\n         :                          +- * HashAggregate (22)\n         :                             +- * HashAggregate (21)\n         :                                +- AQEShuffleRead (20)\n         :                                   +- ShuffleQueryStage (19), Statistics(sizeInBytes=2.1 KiB, rowCount=54)\n         :                                      +- Exchange (18)\n         :                                         +- * HashAggregate (17)\n         :                                            +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- BroadcastExchange (54)\n            +- HashAggregate (53)\n               +- ShuffleQueryStage (52)\n                  +- Exchange (51)\n                     +- * HashAggregate (50)\n                        +- * HashAggregate (49)\n                           +- AQEShuffleRead (48)\n                              +- ShuffleQueryStage (47), Statistics(sizeInBytes=3.8 KiB, rowCount=164)\n                                 +- Exchange (46)\n                                    +- * HashAggregate (45)\n                                       +- * Project (44)\n                                          +- * SortMergeJoin Inner (43)\n                                             :- * Sort (36)\n                                             :  +- AQEShuffleRead (35)\n                                             :     +- ShuffleQueryStage (34), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                             :        +- Exchange (33)\n                                             :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                             +- * Sort (42)\n                                                +- AQEShuffleRead (41)\n                                                   +- ShuffleQueryStage (40), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                      +- Exchange (39)\n                                                         +- * Project (38)\n                                                            +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (90)\n   +- Project (89)\n      +- SortMergeJoin LeftOuter (88)\n         :- LocalLimit (74)\n         :  +- Project (73)\n         :     +- SortMergeJoin LeftOuter (72)\n         :        :- LocalLimit (65)\n         :        :  +- Project (64)\n         :        :     +- SortMergeJoin Inner (63)\n         :        :        :- Sort (59)\n         :        :        :  +- Exchange (58)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (62)\n         :        :           +- HashAggregate (61)\n         :        :              +- Exchange (60)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (71)\n         :           +- HashAggregate (70)\n         :              +- Exchange (69)\n         :                 +- HashAggregate (68)\n         :                    +- HashAggregate (67)\n         :                       +- Exchange (66)\n         :                          +- HashAggregate (17)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- Sort (87)\n            +- HashAggregate (86)\n               +- Exchange (85)\n                  +- HashAggregate (84)\n                     +- HashAggregate (83)\n                        +- Exchange (82)\n                           +- HashAggregate (81)\n                              +- Project (80)\n                                 +- SortMergeJoin Inner (79)\n                                    :- Sort (76)\n                                    :  +- Exchange (75)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                    +- Sort (78)\n                                       +- Exchange (77)\n                                          +- Project (38)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(12) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(13) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(15) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(16) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(17) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(18) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(19) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(20) AQEShuffleRead\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: coalesced\n\n(21) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(22) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(23) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]\n\n(24) ShuffleQueryStage\nOutput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: 7\n\n(25) AQEShuffleRead\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 11]\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(27) BroadcastExchange\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]\n\n(28) BroadcastQueryStage\nOutput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 8\n\n(29) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(30) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(31) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(32) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(33) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(34) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(35) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(36) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(37) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(38) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(39) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(40) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(41) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(42) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(44) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(45) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(46) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(47) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(48) AQEShuffleRead\nInput [2]: [user_id#114, lesson_id#37]\nArguments: coalesced\n\n(49) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(50) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(51) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]\n\n(52) ShuffleQueryStage\nOutput [2]: [user_id#114, count#330L]\nArguments: 9\n\n(53) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(54) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=773]\n\n(55) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(56) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(57) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(58) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(59) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(60) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(61) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(62) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(63) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(64) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(65) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(66) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(67) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(68) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(69) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(70) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(71) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(72) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(73) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(74) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(75) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(76) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(77) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(78) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(79) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(80) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(81) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(82) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(83) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(84) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(85) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(86) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(87) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(88) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(89) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(90) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(91) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#51], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":965,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1012,"metricType":"sum"},{"name":"partition data size","accumulatorId":1013,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1014,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1008,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1009,"metricType":"timing"},{"name":"peak memory","accumulatorId":1007,"metricType":"size"},{"name":"number of output rows","accumulatorId":1006,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1010,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1002,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1003,"metricType":"timing"},{"name":"peak memory","accumulatorId":1001,"metricType":"size"},{"name":"number of output rows","accumulatorId":1000,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1005,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1004,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":988,"metricType":"sum"},{"name":"records read","accumulatorId":985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":979,"metricType":"sum"},{"name":"data size","accumulatorId":977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":993,"metricType":"size"},{"name":"number of partitions","accumulatorId":978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1089,"metricType":"sum"},{"name":"partition data size","accumulatorId":1090,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1091,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1085,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1086,"metricType":"timing"},{"name":"peak memory","accumulatorId":1084,"metricType":"size"},{"name":"number of output rows","accumulatorId":1083,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1088,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1087,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1082,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1081,"metricType":"timing"},{"name":"time to build","accumulatorId":1080,"metricType":"timing"},{"name":"time to collect","accumulatorId":1079,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1078,"metricType":"sum"},{"name":"data size","accumulatorId":1077,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1076,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=773]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1194,"metricType":"sum"},{"name":"partition data size","accumulatorId":1195,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1196,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1190,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1191,"metricType":"timing"},{"name":"peak memory","accumulatorId":1189,"metricType":"size"},{"name":"number of output rows","accumulatorId":1188,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1193,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1192,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1184,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1185,"metricType":"timing"},{"name":"peak memory","accumulatorId":1183,"metricType":"size"},{"name":"number of output rows","accumulatorId":1182,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1187,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1186,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1181,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1179,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1173,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1180,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1174,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1171,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1168,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1177,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1170,"metricType":"sum"},{"name":"records read","accumulatorId":1167,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1165,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1166,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1163,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1169,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1162,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1172,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1161,"metricType":"sum"},{"name":"data size","accumulatorId":1159,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1175,"metricType":"size"},{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1176,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1164,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1178,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1155,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1156,"metricType":"timing"},{"name":"peak memory","accumulatorId":1154,"metricType":"size"},{"name":"number of output rows","accumulatorId":1153,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1158,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1157,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1145,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1139,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1146,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1140,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1137,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1134,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1143,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1136,"metricType":"sum"},{"name":"records read","accumulatorId":1133,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1131,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1132,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1129,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1135,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1128,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1138,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1127,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1141,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1142,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1130,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1144,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":14,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428226,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428322,"Failed":false,"Killed":false,"Accumulables":[{"ID":980,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"local bytes read","Update":"880","Value":"880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1082,"Name":"duration","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1083,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1084,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1086,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1092,"Name":"internal.metrics.executorDeserializeTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1093,"Name":"internal.metrics.executorDeserializeCpuTime","Update":12202132,"Value":12202132,"Internal":true,"Count Failed Values":true},{"ID":1094,"Name":"internal.metrics.executorRunTime","Update":55,"Value":55,"Internal":true,"Count Failed Values":true},{"ID":1095,"Name":"internal.metrics.executorCpuTime","Update":31535318,"Value":31535318,"Internal":true,"Count Failed Values":true},{"ID":1096,"Name":"internal.metrics.resultSize","Update":7163,"Value":7163,"Internal":true,"Count Failed Values":true},{"ID":1098,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1101,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1103,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1104,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1105,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1106,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1107,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":880,"Value":880,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1111,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1113,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1114,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1115,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1116,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1117,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1118,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1119,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":23,"Executor Deserialize CPU Time":12202132,"Executor Run Time":55,"Executor CPU Time":31535318,"Peak Execution Memory":4456432,"Result Size":7163,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":880,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"51\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"55\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"52\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[13],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428214,"Completion Time":1784293428329,"Accumulables":[{"ID":980,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":983,"Name":"local bytes read","Value":"880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":984,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":985,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1082,"Name":"duration","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1083,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1084,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1086,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1087,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1092,"Name":"internal.metrics.executorDeserializeTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1093,"Name":"internal.metrics.executorDeserializeCpuTime","Value":12202132,"Internal":true,"Count Failed Values":true},{"ID":1094,"Name":"internal.metrics.executorRunTime","Value":55,"Internal":true,"Count Failed Values":true},{"ID":1095,"Name":"internal.metrics.executorCpuTime","Value":31535318,"Internal":true,"Count Failed Values":true},{"ID":1096,"Name":"internal.metrics.resultSize","Value":7163,"Internal":true,"Count Failed Values":true},{"ID":1098,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1101,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1103,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1104,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1105,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1106,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1107,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":880,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1111,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1113,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1114,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1115,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1116,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1117,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1118,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1119,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":8,"Completion Time":1784293428331,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1081,16],[1080,4],[1079,154],[1078,11],[1077,1048688]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1194,1],[1196,1],[1195,8064]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1160,200]]}
{"Event":"SparkListenerJobStart","Job ID":9,"Submission Time":1784293428367,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19,21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"38\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15,16],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"70\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[15,16,17,18],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"70\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428371,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428382,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428382,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428472,"Failed":false,"Killed":false,"Accumulables":[{"ID":831,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":834,"Name":"local bytes read","Update":"7703","Value":"7703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":835,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":836,"Name":"records read","Update":"164","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1159,"Name":"data size","Update":"480","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1178,"Name":"shuffle bytes written","Update":"1160","Value":"1160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1179,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1180,"Name":"shuffle write time","Update":"3750850","Value":"3750850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1181,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1182,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1183,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1185,"Name":"time in aggregation build","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1188,"Name":"number of output rows","Update":"164","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1189,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1191,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1197,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1198,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9108182,"Value":9108182,"Internal":true,"Count Failed Values":true},{"ID":1199,"Name":"internal.metrics.executorRunTime","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":1200,"Name":"internal.metrics.executorCpuTime","Update":55296531,"Value":55296531,"Internal":true,"Count Failed Values":true},{"ID":1201,"Name":"internal.metrics.resultSize","Update":8765,"Value":8765,"Internal":true,"Count Failed Values":true},{"ID":1206,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1208,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1209,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1210,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1212,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":7703,"Value":7703,"Internal":true,"Count Failed Values":true},{"ID":1213,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1214,"Name":"internal.metrics.shuffle.read.recordsRead","Update":164,"Value":164,"Internal":true,"Count Failed Values":true},{"ID":1215,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1216,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1217,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1218,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1219,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1220,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1221,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1222,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1223,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1224,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1225,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1160,"Value":1160,"Internal":true,"Count Failed Values":true},{"ID":1226,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1227,"Name":"internal.metrics.shuffle.write.writeTime","Update":3750850,"Value":3750850,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":9108182,"Executor Run Time":59,"Executor CPU Time":55296531,"Peak Execution Memory":524288,"Result Size":8765,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":7703,"Total Records Read":164,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1160,"Shuffle Write Time":3750850,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"70\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428371,"Completion Time":1784293428473,"Accumulables":[{"ID":831,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":834,"Name":"local bytes read","Value":"7703","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":835,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":836,"Name":"records read","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1159,"Name":"data size","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1178,"Name":"shuffle bytes written","Value":"1160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1179,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1180,"Name":"shuffle write time","Value":"3750850","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1181,"Name":"duration","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1182,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1183,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1185,"Name":"time in aggregation build","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1188,"Name":"number of output rows","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1189,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1191,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1197,"Name":"internal.metrics.executorDeserializeTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":1198,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9108182,"Internal":true,"Count Failed Values":true},{"ID":1199,"Name":"internal.metrics.executorRunTime","Value":59,"Internal":true,"Count Failed Values":true},{"ID":1200,"Name":"internal.metrics.executorCpuTime","Value":55296531,"Internal":true,"Count Failed Values":true},{"ID":1201,"Name":"internal.metrics.resultSize","Value":8765,"Internal":true,"Count Failed Values":true},{"ID":1206,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1208,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1209,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1210,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1211,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1212,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":7703,"Internal":true,"Count Failed Values":true},{"ID":1213,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1214,"Name":"internal.metrics.shuffle.read.recordsRead","Value":164,"Internal":true,"Count Failed Values":true},{"ID":1215,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1216,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1217,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1218,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1219,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1220,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1221,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1222,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1223,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1224,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1225,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1160,"Internal":true,"Count Failed Values":true},{"ID":1226,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1227,"Name":"internal.metrics.shuffle.write.writeTime","Value":3750850,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":9,"Completion Time":1784293428474,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (93)\n+- == Current Plan ==\n   CollectLimit (59)\n   +- Project (58)\n      +- BroadcastHashJoin LeftOuter BuildRight (57)\n         :- LocalLimit (31)\n         :  +- Project (30)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (29)\n         :        :- LocalLimit (15)\n         :        :  +- Project (14)\n         :        :     +- BroadcastHashJoin Inner BuildRight (13)\n         :        :        :- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :        :  +- Exchange (3)\n         :        :        :     +- * Project (2)\n         :        :        :        +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- BroadcastQueryStage (12), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n         :        :           +- BroadcastExchange (11)\n         :        :              +- * HashAggregate (10)\n         :        :                 +- AQEShuffleRead (9)\n         :        :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                       +- Exchange (7)\n         :        :                          +- * HashAggregate (6)\n         :        :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- BroadcastQueryStage (28), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n         :           +- BroadcastExchange (27)\n         :              +- * HashAggregate (26)\n         :                 +- AQEShuffleRead (25)\n         :                    +- ShuffleQueryStage (24), Statistics(sizeInBytes=440.0 B, rowCount=11)\n         :                       +- Exchange (23)\n         :                          +- * HashAggregate (22)\n         :                             +- * HashAggregate (21)\n         :                                +- AQEShuffleRead (20)\n         :                                   +- ShuffleQueryStage (19), Statistics(sizeInBytes=2.1 KiB, rowCount=54)\n         :                                      +- Exchange (18)\n         :                                         +- * HashAggregate (17)\n         :                                            +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- BroadcastQueryStage (56)\n            +- BroadcastExchange (55)\n               +- * HashAggregate (54)\n                  +- AQEShuffleRead (53)\n                     +- ShuffleQueryStage (52), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                        +- Exchange (51)\n                           +- * HashAggregate (50)\n                              +- * HashAggregate (49)\n                                 +- AQEShuffleRead (48)\n                                    +- ShuffleQueryStage (47), Statistics(sizeInBytes=3.8 KiB, rowCount=164)\n                                       +- Exchange (46)\n                                          +- * HashAggregate (45)\n                                             +- * Project (44)\n                                                +- * SortMergeJoin Inner (43)\n                                                   :- * Sort (36)\n                                                   :  +- AQEShuffleRead (35)\n                                                   :     +- ShuffleQueryStage (34), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                                   :        +- Exchange (33)\n                                                   :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                                   +- * Sort (42)\n                                                      +- AQEShuffleRead (41)\n                                                         +- ShuffleQueryStage (40), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                            +- Exchange (39)\n                                                               +- * Project (38)\n                                                                  +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (92)\n   +- Project (91)\n      +- SortMergeJoin LeftOuter (90)\n         :- LocalLimit (76)\n         :  +- Project (75)\n         :     +- SortMergeJoin LeftOuter (74)\n         :        :- LocalLimit (67)\n         :        :  +- Project (66)\n         :        :     +- SortMergeJoin Inner (65)\n         :        :        :- Sort (61)\n         :        :        :  +- Exchange (60)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (64)\n         :        :           +- HashAggregate (63)\n         :        :              +- Exchange (62)\n         :        :                 +- HashAggregate (6)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         :        +- Sort (73)\n         :           +- HashAggregate (72)\n         :              +- Exchange (71)\n         :                 +- HashAggregate (70)\n         :                    +- HashAggregate (69)\n         :                       +- Exchange (68)\n         :                          +- HashAggregate (17)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (16)\n         +- Sort (89)\n            +- HashAggregate (88)\n               +- Exchange (87)\n                  +- HashAggregate (86)\n                     +- HashAggregate (85)\n                        +- Exchange (84)\n                           +- HashAggregate (83)\n                              +- Project (82)\n                                 +- SortMergeJoin Inner (81)\n                                    :- Sort (78)\n                                    :  +- Exchange (77)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (32)\n                                    +- Sort (80)\n                                       +- Exchange (79)\n                                          +- Project (38)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(7) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(11) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(12) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(13) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(15) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(16) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(17) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(18) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(19) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(20) AQEShuffleRead\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: coalesced\n\n(21) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(22) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(23) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]\n\n(24) ShuffleQueryStage\nOutput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: 7\n\n(25) AQEShuffleRead\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 11]\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(27) BroadcastExchange\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]\n\n(28) BroadcastQueryStage\nOutput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 8\n\n(29) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(30) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(31) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(32) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(33) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(34) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(35) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(36) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(37) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(38) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(39) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(40) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(41) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(42) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(44) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(45) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(46) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(47) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(48) AQEShuffleRead\nInput [2]: [user_id#114, lesson_id#37]\nArguments: coalesced\n\n(49) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(50) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(51) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]\n\n(52) ShuffleQueryStage\nOutput [2]: [user_id#114, count#330L]\nArguments: 9\n\n(53) AQEShuffleRead\nInput [2]: [user_id#114, count#330L]\nArguments: coalesced\n\n(54) HashAggregate [codegen id : 13]\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(55) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=860]\n\n(56) BroadcastQueryStage\nOutput [2]: [user_id#114, possible_lessons#94L]\nArguments: 10\n\n(57) BroadcastHashJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(58) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(59) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(60) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(61) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(62) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(63) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(64) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(65) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(66) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(67) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(68) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(69) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(70) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(71) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(72) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(73) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(74) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(75) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(76) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(77) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(78) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(79) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(80) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(81) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(82) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(83) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(84) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(85) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(86) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(87) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(88) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(89) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(90) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(91) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(92) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(93) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#51], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":965,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1012,"metricType":"sum"},{"name":"partition data size","accumulatorId":1013,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1014,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1008,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1009,"metricType":"timing"},{"name":"peak memory","accumulatorId":1007,"metricType":"size"},{"name":"number of output rows","accumulatorId":1006,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1010,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1002,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1003,"metricType":"timing"},{"name":"peak memory","accumulatorId":1001,"metricType":"size"},{"name":"number of output rows","accumulatorId":1000,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1005,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1004,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":988,"metricType":"sum"},{"name":"records read","accumulatorId":985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":979,"metricType":"sum"},{"name":"data size","accumulatorId":977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":993,"metricType":"size"},{"name":"number of partitions","accumulatorId":978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1089,"metricType":"sum"},{"name":"partition data size","accumulatorId":1090,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1091,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1085,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1086,"metricType":"timing"},{"name":"peak memory","accumulatorId":1084,"metricType":"size"},{"name":"number of output rows","accumulatorId":1083,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1088,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1087,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1082,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1081,"metricType":"timing"},{"name":"time to build","accumulatorId":1080,"metricType":"timing"},{"name":"time to collect","accumulatorId":1079,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1078,"metricType":"sum"},{"name":"data size","accumulatorId":1077,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1076,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=860]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1194,"metricType":"sum"},{"name":"partition data size","accumulatorId":1195,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1196,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1190,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1191,"metricType":"timing"},{"name":"peak memory","accumulatorId":1189,"metricType":"size"},{"name":"number of output rows","accumulatorId":1188,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1193,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1192,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1184,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1185,"metricType":"timing"},{"name":"peak memory","accumulatorId":1183,"metricType":"size"},{"name":"number of output rows","accumulatorId":1182,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1187,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1186,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1181,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1179,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1173,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1180,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1174,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1171,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1168,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1177,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1170,"metricType":"sum"},{"name":"records read","accumulatorId":1167,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1165,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1166,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1163,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1169,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1162,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1172,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1161,"metricType":"sum"},{"name":"data size","accumulatorId":1159,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1175,"metricType":"size"},{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1176,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1164,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1178,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1265,"metricType":"sum"},{"name":"partition data size","accumulatorId":1266,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1267,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1261,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1262,"metricType":"timing"},{"name":"peak memory","accumulatorId":1260,"metricType":"size"},{"name":"number of output rows","accumulatorId":1259,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1264,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1263,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1258,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1257,"metricType":"timing"},{"name":"time to build","accumulatorId":1256,"metricType":"timing"},{"name":"time to collect","accumulatorId":1255,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1254,"metricType":"sum"},{"name":"data size","accumulatorId":1253,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1252,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1250,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1244,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1251,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1245,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1242,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1239,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1248,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1241,"metricType":"sum"},{"name":"records read","accumulatorId":1238,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1236,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1237,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1234,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1240,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1233,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1243,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1232,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1246,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1247,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1235,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1249,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1265,1],[1267,1],[1266,1216]]}
{"Event":"SparkListenerJobStart","Job ID":10,"Submission Time":1784293428508,"Stage Infos":[{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"20\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"33\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19,21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":20,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"38\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"21\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"35\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[19,20],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"56\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"70\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[21],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"71\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"75\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[19,20,21,22,23],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"71\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 937147bc-057a-4499-bba2-a05b5aa70dbf)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"71\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"75\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428509,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"71\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 937147bc-057a-4499-bba2-a05b5aa70dbf)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":23,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428514,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":23,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428514,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428556,"Failed":false,"Killed":false,"Accumulables":[{"ID":1162,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1165,"Name":"local bytes read","Update":"1160","Value":"1160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1166,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1167,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1258,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1259,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1260,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1262,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1263,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1268,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8482917,"Value":8482917,"Internal":true,"Count Failed Values":true},{"ID":1270,"Name":"internal.metrics.executorRunTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":1271,"Name":"internal.metrics.executorCpuTime","Update":18421772,"Value":18421772,"Internal":true,"Count Failed Values":true},{"ID":1272,"Name":"internal.metrics.resultSize","Update":10247,"Value":10247,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1160,"Value":1160,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":8482917,"Executor Run Time":24,"Executor CPU Time":18421772,"Peak Execution Memory":4456432,"Result Size":10247,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1160,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"71\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"75\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428509,"Completion Time":1784293428557,"Accumulables":[{"ID":1162,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1165,"Name":"local bytes read","Value":"1160","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1166,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1167,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1258,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1259,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1260,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1262,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1263,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1268,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8482917,"Internal":true,"Count Failed Values":true},{"ID":1270,"Name":"internal.metrics.executorRunTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":1271,"Name":"internal.metrics.executorCpuTime","Value":18421772,"Internal":true,"Count Failed Values":true},{"ID":1272,"Name":"internal.metrics.resultSize","Value":10247,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1160,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":10,"Completion Time":1784293428558,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1257,1],[1256,2],[1255,68],[1254,20],[1253,1048736]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (94)\n+- == Final Plan ==\n   CollectLimit (60)\n   +- * Project (59)\n      +- * BroadcastHashJoin LeftOuter BuildRight (58)\n         :- * LocalLimit (32)\n         :  +- * Project (31)\n         :     +- * BroadcastHashJoin LeftOuter BuildRight (30)\n         :        :- * LocalLimit (16)\n         :        :  +- * Project (15)\n         :        :     +- * BroadcastHashJoin Inner BuildRight (14)\n         :        :        :- AQEShuffleRead (5)\n         :        :        :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=640.0 B, rowCount=20)\n         :        :        :     +- Exchange (3)\n         :        :        :        +- * Project (2)\n         :        :        :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- BroadcastQueryStage (13), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n         :        :           +- BroadcastExchange (12)\n         :        :              +- * HashAggregate (11)\n         :        :                 +- AQEShuffleRead (10)\n         :        :                    +- ShuffleQueryStage (9), Statistics(sizeInBytes=480.0 B, rowCount=20)\n         :        :                       +- Exchange (8)\n         :        :                          +- * HashAggregate (7)\n         :        :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         :        +- BroadcastQueryStage (29), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n         :           +- BroadcastExchange (28)\n         :              +- * HashAggregate (27)\n         :                 +- AQEShuffleRead (26)\n         :                    +- ShuffleQueryStage (25), Statistics(sizeInBytes=440.0 B, rowCount=11)\n         :                       +- Exchange (24)\n         :                          +- * HashAggregate (23)\n         :                             +- * HashAggregate (22)\n         :                                +- AQEShuffleRead (21)\n         :                                   +- ShuffleQueryStage (20), Statistics(sizeInBytes=2.1 KiB, rowCount=54)\n         :                                      +- Exchange (19)\n         :                                         +- * HashAggregate (18)\n         :                                            +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (17)\n         +- BroadcastQueryStage (57), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n            +- BroadcastExchange (56)\n               +- * HashAggregate (55)\n                  +- AQEShuffleRead (54)\n                     +- ShuffleQueryStage (53), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                        +- Exchange (52)\n                           +- * HashAggregate (51)\n                              +- * HashAggregate (50)\n                                 +- AQEShuffleRead (49)\n                                    +- ShuffleQueryStage (48), Statistics(sizeInBytes=3.8 KiB, rowCount=164)\n                                       +- Exchange (47)\n                                          +- * HashAggregate (46)\n                                             +- * Project (45)\n                                                +- * SortMergeJoin Inner (44)\n                                                   :- * Sort (37)\n                                                   :  +- AQEShuffleRead (36)\n                                                   :     +- ShuffleQueryStage (35), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                                   :        +- Exchange (34)\n                                                   :           +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (33)\n                                                   +- * Sort (43)\n                                                      +- AQEShuffleRead (42)\n                                                         +- ShuffleQueryStage (41), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                                            +- Exchange (40)\n                                                               +- * Project (39)\n                                                                  +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (38)\n+- == Initial Plan ==\n   CollectLimit (93)\n   +- Project (92)\n      +- SortMergeJoin LeftOuter (91)\n         :- LocalLimit (77)\n         :  +- Project (76)\n         :     +- SortMergeJoin LeftOuter (75)\n         :        :- LocalLimit (68)\n         :        :  +- Project (67)\n         :        :     +- SortMergeJoin Inner (66)\n         :        :        :- Sort (62)\n         :        :        :  +- Exchange (61)\n         :        :        :     +- Project (2)\n         :        :        :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n         :        :        +- Sort (65)\n         :        :           +- HashAggregate (64)\n         :        :              +- Exchange (63)\n         :        :                 +- HashAggregate (7)\n         :        :                    +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         :        +- Sort (74)\n         :           +- HashAggregate (73)\n         :              +- Exchange (72)\n         :                 +- HashAggregate (71)\n         :                    +- HashAggregate (70)\n         :                       +- Exchange (69)\n         :                          +- HashAggregate (18)\n         :                             +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (17)\n         +- Sort (90)\n            +- HashAggregate (89)\n               +- Exchange (88)\n                  +- HashAggregate (87)\n                     +- HashAggregate (86)\n                        +- Exchange (85)\n                           +- HashAggregate (84)\n                              +- Project (83)\n                                 +- SortMergeJoin Inner (82)\n                                    :- Sort (79)\n                                    :  +- Exchange (78)\n                                    :     +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (33)\n                                    +- Sort (81)\n                                       +- Exchange (80)\n                                          +- Project (39)\n                                             +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (38)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#0, name#1]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,name:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#0 AS user_id#10, name#1]\nInput [2]: [id#0, name#1]\n\n(3) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]\n\n(4) ShuffleQueryStage\nOutput [2]: [user_id#10, name#1]\nArguments: 0\n\n(5) AQEShuffleRead\nInput [2]: [user_id#10, name#1]\nArguments: local\n\n(6) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(7) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [1]: [user_id#43]\nFunctions [1]: [partial_count(course_id#44)]\nAggregate Attributes [1]: [count#318L]\nResults [2]: [user_id#43, count#319L]\n\n(8) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]\n\n(9) ShuffleQueryStage\nOutput [2]: [user_id#43, count#319L]\nArguments: 1\n\n(10) AQEShuffleRead\nInput [2]: [user_id#43, count#319L]\nArguments: coalesced\n\n(11) HashAggregate [codegen id : 6]\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(12) BroadcastExchange\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]\n\n(13) BroadcastQueryStage\nOutput [2]: [user_id#43, enrollment_count#63L]\nArguments: 5\n\n(14) BroadcastHashJoin [codegen id : 14]\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(15) Project [codegen id : 14]\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(16) LocalLimit [codegen id : 14]\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(17) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>\n\n(18) HashAggregate [codegen id : 3]\nInput [3]: [user_id#51, lesson_id#52, viewed_at#53]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(19) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]\n\n(20) ShuffleQueryStage\nOutput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: 2\n\n(21) AQEShuffleRead\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: coalesced\n\n(22) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(23) HashAggregate [codegen id : 10]\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(24) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]\n\n(25) ShuffleQueryStage\nOutput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: 7\n\n(26) AQEShuffleRead\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 11]\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(28) BroadcastExchange\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]\n\n(29) BroadcastQueryStage\nOutput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 8\n\n(30) BroadcastHashJoin [codegen id : 14]\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(31) Project [codegen id : 14]\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(32) LocalLimit [codegen id : 14]\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(33) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [user_id#114, course_id#115]\nPushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(34) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]\n\n(35) ShuffleQueryStage\nOutput [2]: [user_id#114, course_id#115]\nArguments: 3\n\n(36) AQEShuffleRead\nInput [2]: [user_id#114, course_id#115]\nArguments: coalesced\n\n(37) Sort [codegen id : 7]\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(38) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(39) Project [codegen id : 5]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(40) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]\n\n(41) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 4\n\n(42) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(43) Sort [codegen id : 8]\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(44) SortMergeJoin [codegen id : 9]\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(45) Project [codegen id : 9]\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(46) HashAggregate [codegen id : 9]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(47) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]\n\n(48) ShuffleQueryStage\nOutput [2]: [user_id#114, lesson_id#37]\nArguments: 6\n\n(49) AQEShuffleRead\nInput [2]: [user_id#114, lesson_id#37]\nArguments: coalesced\n\n(50) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(51) HashAggregate [codegen id : 12]\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(52) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]\n\n(53) ShuffleQueryStage\nOutput [2]: [user_id#114, count#330L]\nArguments: 9\n\n(54) AQEShuffleRead\nInput [2]: [user_id#114, count#330L]\nArguments: coalesced\n\n(55) HashAggregate [codegen id : 13]\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(56) BroadcastExchange\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=860]\n\n(57) BroadcastQueryStage\nOutput [2]: [user_id#114, possible_lessons#94L]\nArguments: 10\n\n(58) BroadcastHashJoin [codegen id : 14]\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(59) Project [codegen id : 14]\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(60) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(61) Exchange\nInput [2]: [user_id#10, name#1]\nArguments: hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=104]\n\n(62) Sort\nInput [2]: [user_id#10, name#1]\nArguments: [user_id#10 ASC NULLS FIRST], false, 0\n\n(63) Exchange\nInput [2]: [user_id#43, count#319L]\nArguments: hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=100]\n\n(64) HashAggregate\nInput [2]: [user_id#43, count#319L]\nKeys [1]: [user_id#43]\nFunctions [1]: [count(course_id#44)]\nAggregate Attributes [1]: [count(course_id#44)#62L]\nResults [2]: [user_id#43, count(course_id#44)#62L AS enrollment_count#63L]\n\n(65) Sort\nInput [2]: [user_id#43, enrollment_count#63L]\nArguments: [user_id#43 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#43]\nJoin type: Inner\nJoin condition: None\n\n(67) Project\nOutput [3]: [user_id#10, name#1, enrollment_count#63L]\nInput [4]: [user_id#10, name#1, user_id#43, enrollment_count#63L]\n\n(68) LocalLimit\nInput [3]: [user_id#10, name#1, enrollment_count#63L]\nArguments: 21\n\n(69) Exchange\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nArguments: hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=110]\n\n(70) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [2]: [user_id#51, lesson_id#52]\nFunctions [2]: [merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [2]: [min(viewed_at#53)#71, max(viewed_at#53)#73]\nResults [4]: [user_id#51, lesson_id#52, min#321, max#323]\n\n(71) HashAggregate\nInput [4]: [user_id#51, lesson_id#52, min#321, max#323]\nKeys [1]: [user_id#51]\nFunctions [3]: [merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, min#321, max#323, count#326L]\n\n(72) Exchange\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=114]\n\n(73) HashAggregate\nInput [4]: [user_id#51, min#321, max#323, count#326L]\nKeys [1]: [user_id#51]\nFunctions [3]: [min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)]\nAggregate Attributes [3]: [min(viewed_at#53)#71, max(viewed_at#53)#73, count(lesson_id#52)#75L]\nResults [4]: [user_id#51, count(lesson_id#52)#75L AS viewed_lessons_count#70L, min(viewed_at#53)#71 AS first_view#72, max(viewed_at#53)#73 AS last_view#74]\n\n(74) Sort\nInput [4]: [user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: [user_id#51 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#51]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nInput [7]: [user_id#10, name#1, enrollment_count#63L, user_id#51, viewed_lessons_count#70L, first_view#72, last_view#74]\n\n(77) LocalLimit\nInput [6]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]\nArguments: 21\n\n(78) Exchange\nInput [2]: [user_id#114, course_id#115]\nArguments: hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=123]\n\n(79) Sort\nInput [2]: [user_id#114, course_id#115]\nArguments: [course_id#115 ASC NULLS FIRST], false, 0\n\n(80) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=124]\n\n(81) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(82) SortMergeJoin\nLeft keys [1]: [course_id#115]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(83) Project\nOutput [2]: [user_id#114, lesson_id#37]\nInput [4]: [user_id#114, course_id#115, lesson_id#37, course_id#30]\n\n(84) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(85) Exchange\nInput [2]: [user_id#114, lesson_id#37]\nArguments: hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(86) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [2]: [user_id#114, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#114, lesson_id#37]\n\n(87) HashAggregate\nInput [2]: [user_id#114, lesson_id#37]\nKeys [1]: [user_id#114]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count#330L]\n\n(88) Exchange\nInput [2]: [user_id#114, count#330L]\nArguments: hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=135]\n\n(89) HashAggregate\nInput [2]: [user_id#114, count#330L]\nKeys [1]: [user_id#114]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#95L]\nResults [2]: [user_id#114, count(lesson_id#37)#95L AS possible_lessons#94L]\n\n(90) Sort\nInput [2]: [user_id#114, possible_lessons#94L]\nArguments: [user_id#114 ASC NULLS FIRST], false, 0\n\n(91) SortMergeJoin\nLeft keys [1]: [user_id#10]\nRight keys [1]: [user_id#114]\nJoin type: LeftOuter\nJoin condition: None\n\n(92) Project\nOutput [7]: [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]\nInput [8]: [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74, user_id#114, possible_lessons#94L]\n\n(93) CollectLimit\nInput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: 21\n\n(94) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count)#307, toprettystring(first_view)#308, toprettystring(last_view)#309, toprettystring(engagement_rate)#310]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (14)","simpleString":"WholeStageCodegen (14)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#10, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(enrollment_count#63L, Some(Europe/Samara)) AS toprettystring(enrollment_count)#306, toprettystring(viewed_lessons_count#70L, Some(Europe/Samara)) AS toprettystring(viewed_lessons_count)#307, toprettystring(first_view#72, Some(Europe/Samara)) AS toprettystring(first_view)#308, toprettystring(last_view#74, Some(Europe/Samara)) AS toprettystring(last_view)#309, toprettystring(concat(cast(round(((cast(coalesce(viewed_lessons_count#70L, 0) as double) * 100.0) / cast(possible_lessons#94L as double)), 2) as string), %), Some(Europe/Samara)) AS toprettystring(engagement_rate)#310]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#114], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L, viewed_lessons_count#70L, first_view#72, last_view#74]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#51], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [user_id#10, name#1, enrollment_count#63L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#10], [user_id#43], Inner, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=149]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#0 AS user_id#10, name#1]","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [id#0,name#1] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":52,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":339,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":337,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":331,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":338,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":332,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":329,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":326,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":335,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":328,"metricType":"sum"},{"name":"records read","accumulatorId":325,"metricType":"sum"},{"name":"local bytes read","accumulatorId":323,"metricType":"size"},{"name":"fetch wait time","accumulatorId":324,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":321,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":327,"metricType":"sum"},{"name":"local blocks read","accumulatorId":320,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":330,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":319,"metricType":"sum"},{"name":"data size","accumulatorId":317,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":333,"metricType":"size"},{"name":"number of partitions","accumulatorId":318,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":334,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":322,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":336,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1327,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=441]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[count(course_id#44)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=161]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#43], functions=[partial_count(course_id#44)])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":90,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":86,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":87,"metricType":"timing"},{"name":"peak memory","accumulatorId":85,"metricType":"size"},{"name":"number of output rows","accumulatorId":84,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":89,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":88,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":371,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":369,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":363,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":370,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":364,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":361,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":358,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":367,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":360,"metricType":"sum"},{"name":"records read","accumulatorId":357,"metricType":"sum"},{"name":"local bytes read","accumulatorId":355,"metricType":"size"},{"name":"fetch wait time","accumulatorId":356,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":353,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":359,"metricType":"sum"},{"name":"local blocks read","accumulatorId":352,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":362,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":351,"metricType":"sum"},{"name":"data size","accumulatorId":349,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":365,"metricType":"size"},{"name":"number of partitions","accumulatorId":350,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":366,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":354,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":368,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":777,"metricType":"sum"},{"name":"partition data size","accumulatorId":778,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":773,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":774,"metricType":"timing"},{"name":"peak memory","accumulatorId":772,"metricType":"size"},{"name":"number of output rows","accumulatorId":771,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":776,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":775,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":770,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":769,"metricType":"timing"},{"name":"time to build","accumulatorId":768,"metricType":"timing"},{"name":"time to collect","accumulatorId":767,"metricType":"timing"},{"name":"number of output rows","accumulatorId":766,"metricType":"sum"},{"name":"data size","accumulatorId":765,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1326,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=702]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#52)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=581]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#52)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=183]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51, lesson_id#52], functions=[partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51,lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:int,lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":162,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":158,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":159,"metricType":"timing"},{"name":"peak memory","accumulatorId":157,"metricType":"size"},{"name":"number of output rows","accumulatorId":156,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":161,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":160,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":437,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":435,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":429,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":436,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":427,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":424,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":433,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"records read","accumulatorId":423,"metricType":"sum"},{"name":"local bytes read","accumulatorId":421,"metricType":"size"},{"name":"fetch wait time","accumulatorId":422,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":419,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":425,"metricType":"sum"},{"name":"local blocks read","accumulatorId":418,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":417,"metricType":"sum"},{"name":"data size","accumulatorId":415,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":431,"metricType":"size"},{"name":"number of partitions","accumulatorId":416,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":420,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":434,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1012,"metricType":"sum"},{"name":"partition data size","accumulatorId":1013,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1014,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1008,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1009,"metricType":"timing"},{"name":"peak memory","accumulatorId":1007,"metricType":"size"},{"name":"number of output rows","accumulatorId":1006,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1011,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1010,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1002,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1003,"metricType":"timing"},{"name":"peak memory","accumulatorId":1001,"metricType":"size"},{"name":"number of output rows","accumulatorId":1000,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1005,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1004,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":988,"metricType":"sum"},{"name":"records read","accumulatorId":985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":979,"metricType":"sum"},{"name":"data size","accumulatorId":977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":993,"metricType":"size"},{"name":"number of partitions","accumulatorId":978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1089,"metricType":"sum"},{"name":"partition data size","accumulatorId":1090,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1091,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1085,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1086,"metricType":"timing"},{"name":"peak memory","accumulatorId":1084,"metricType":"size"},{"name":"number of output rows","accumulatorId":1083,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1088,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1087,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1082,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1081,"metricType":"timing"},{"name":"time to build","accumulatorId":1080,"metricType":"timing"},{"name":"time to collect","accumulatorId":1079,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1078,"metricType":"sum"},{"name":"data size","accumulatorId":1077,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1325,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=860]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, 200), ENSURE_REQUIREMENTS, [plan_id=761]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#114, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=497]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#114, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [user_id#114, lesson_id#37]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#115], [course_id#30], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#115 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#115, 200), ENSURE_REQUIREMENTS, [plan_id=223]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#114,course_id#115] PushedFilters: [*IsNotNull(course_id), *IsNotNull(user_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":261,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":536,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":534,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":528,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":535,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":529,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":526,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":523,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":532,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":525,"metricType":"sum"},{"name":"records read","accumulatorId":522,"metricType":"sum"},{"name":"local bytes read","accumulatorId":520,"metricType":"size"},{"name":"fetch wait time","accumulatorId":521,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":518,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":524,"metricType":"sum"},{"name":"local blocks read","accumulatorId":517,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":527,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":516,"metricType":"sum"},{"name":"data size","accumulatorId":514,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":530,"metricType":"size"},{"name":"number of partitions","accumulatorId":515,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":531,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":519,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":533,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":863,"metricType":"sum"},{"name":"partition data size","accumulatorId":864,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":865,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":860,"metricType":"timing"},{"name":"peak memory","accumulatorId":861,"metricType":"size"},{"name":"spill size","accumulatorId":862,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":859,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=230]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":287,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":562,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":560,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":554,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":561,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":555,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":552,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":549,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":558,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":551,"metricType":"sum"},{"name":"records read","accumulatorId":548,"metricType":"sum"},{"name":"local bytes read","accumulatorId":546,"metricType":"size"},{"name":"fetch wait time","accumulatorId":547,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":544,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":550,"metricType":"sum"},{"name":"local blocks read","accumulatorId":543,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":553,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":542,"metricType":"sum"},{"name":"data size","accumulatorId":540,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":556,"metricType":"size"},{"name":"number of partitions","accumulatorId":541,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":557,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":545,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":559,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":870,"metricType":"sum"},{"name":"partition data size","accumulatorId":871,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":872,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":867,"metricType":"timing"},{"name":"peak memory","accumulatorId":868,"metricType":"size"},{"name":"spill size","accumulatorId":869,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":866,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":857,"metricType":"sum"},{"name":"spill size","accumulatorId":858,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":853,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":854,"metricType":"timing"},{"name":"peak memory","accumulatorId":852,"metricType":"size"},{"name":"number of output rows","accumulatorId":851,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":856,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":855,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":850,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":848,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":842,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":849,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":843,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":840,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":837,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":846,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":839,"metricType":"sum"},{"name":"records read","accumulatorId":836,"metricType":"sum"},{"name":"local bytes read","accumulatorId":834,"metricType":"size"},{"name":"fetch wait time","accumulatorId":835,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":832,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":838,"metricType":"sum"},{"name":"local blocks read","accumulatorId":831,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":841,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":830,"metricType":"sum"},{"name":"data size","accumulatorId":828,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":844,"metricType":"size"},{"name":"number of partitions","accumulatorId":829,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":845,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":833,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":847,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1194,"metricType":"sum"},{"name":"partition data size","accumulatorId":1195,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1196,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1190,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1191,"metricType":"timing"},{"name":"peak memory","accumulatorId":1189,"metricType":"size"},{"name":"number of output rows","accumulatorId":1188,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1193,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1192,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1184,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1185,"metricType":"timing"},{"name":"peak memory","accumulatorId":1183,"metricType":"size"},{"name":"number of output rows","accumulatorId":1182,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1187,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1186,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1181,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1179,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1173,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1180,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1174,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1171,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1168,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1177,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1170,"metricType":"sum"},{"name":"records read","accumulatorId":1167,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1165,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1166,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1163,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1169,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1162,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1172,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1161,"metricType":"sum"},{"name":"data size","accumulatorId":1159,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1175,"metricType":"size"},{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1176,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1164,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1178,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1265,"metricType":"sum"},{"name":"partition data size","accumulatorId":1266,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1267,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1261,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1262,"metricType":"timing"},{"name":"peak memory","accumulatorId":1260,"metricType":"size"},{"name":"number of output rows","accumulatorId":1259,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1264,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1263,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1258,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1257,"metricType":"timing"},{"name":"time to build","accumulatorId":1256,"metricType":"timing"},{"name":"time to collect","accumulatorId":1255,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1254,"metricType":"sum"},{"name":"data size","accumulatorId":1253,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1324,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1323,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1321,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1315,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1322,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1316,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1313,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1310,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1319,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1312,"metricType":"sum"},{"name":"records read","accumulatorId":1309,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1307,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1308,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1305,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1311,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1304,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1314,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1303,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1317,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1318,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1306,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1320,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1327,1]]}
{"Event":"SparkListenerJobStart","Job ID":11,"Submission Time":1784293428631,"Stage Infos":[{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"118\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"95\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"117\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[24,25],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"118\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"95\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"117\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428633,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":25,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428638,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":25,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293428638,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293428737,"Failed":false,"Killed":false,"Accumulables":[{"ID":320,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":323,"Name":"local bytes read","Update":"1327","Value":"1327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"duration","Update":"42","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":1329,"Name":"internal.metrics.executorDeserializeCpuTime","Update":12604946,"Value":12604946,"Internal":true,"Count Failed Values":true},{"ID":1330,"Name":"internal.metrics.executorRunTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":1331,"Name":"internal.metrics.executorCpuTime","Update":44991721,"Value":44991721,"Internal":true,"Count Failed Values":true},{"ID":1332,"Name":"internal.metrics.resultSize","Update":5009,"Value":5009,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.peakExecutionMemory","Update":2240,"Value":2240,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1341,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1342,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1343,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1327,"Value":1327,"Internal":true,"Count Failed Values":true},{"ID":1344,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1346,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":12604946,"Executor Run Time":75,"Executor CPU Time":44991721,"Peak Execution Memory":2240,"Result Size":5009,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1327,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"118\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[38],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"95\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"117\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293428633,"Completion Time":1784293428738,"Accumulables":[{"ID":320,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":323,"Name":"local bytes read","Value":"1327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":324,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":325,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1323,"Name":"duration","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1324,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1325,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":1329,"Name":"internal.metrics.executorDeserializeCpuTime","Value":12604946,"Internal":true,"Count Failed Values":true},{"ID":1330,"Name":"internal.metrics.executorRunTime","Value":75,"Internal":true,"Count Failed Values":true},{"ID":1331,"Name":"internal.metrics.executorCpuTime","Value":44991721,"Internal":true,"Count Failed Values":true},{"ID":1332,"Name":"internal.metrics.resultSize","Value":5009,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.peakExecutionMemory","Value":2240,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1341,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1342,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1343,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1327,"Internal":true,"Count Failed Values":true},{"ID":1344,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1346,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":11,"Completion Time":1784293428738,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1784293429214,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (33)\n+- CollectLimit (32)\n   +- Project (31)\n      +- SortMergeJoin Inner (30)\n         :- Project (14)\n         :  +- SortMergeJoin Inner (13)\n         :     :- Sort (4)\n         :     :  +- Exchange (3)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (12)\n         :        +- HashAggregate (11)\n         :           +- Exchange (10)\n         :              +- HashAggregate (9)\n         :                 +- HashAggregate (8)\n         :                    +- Exchange (7)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (29)\n            +- HashAggregate (28)\n               +- Exchange (27)\n                  +- HashAggregate (26)\n                     +- HashAggregate (25)\n                        +- HashAggregate (24)\n                           +- Project (23)\n                              +- SortMergeJoin LeftOuter (22)\n                                 :- Sort (18)\n                                 :  +- Exchange (17)\n                                 :     +- Project (16)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (15)\n                                 +- Sort (21)\n                                    +- Exchange (20)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (19)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1] \nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(4) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1] \nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(8) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(9) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(10) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(11) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(12) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(13) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(14) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(15) Scan JDBCRelation(raw.lessons) [numPartitions=1] \nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(16) Project\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(17) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(18) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(19) Scan JDBCRelation(raw.lesson_views) [numPartitions=1] \nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(20) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(21) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(22) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(24) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(25) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(26) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(27) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(28) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(29) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(31) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(32) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(33) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#30], Inner","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#44], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#24 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1410,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1404,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1411,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1405,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1402,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1399,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1408,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1401,"metricType":"sum"},{"name":"records read","accumulatorId":1398,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1396,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1397,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1394,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1400,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1393,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1403,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1392,"metricType":"sum"},{"name":"data size","accumulatorId":1390,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1406,"metricType":"size"},{"name":"number of partitions","accumulatorId":1391,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1407,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1395,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1409,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1387,"metricType":"timing"},{"name":"peak memory","accumulatorId":1388,"metricType":"size"},{"name":"spill size","accumulatorId":1389,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#44 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1476,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1470,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1477,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1471,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1468,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1465,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1474,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1467,"metricType":"sum"},{"name":"records read","accumulatorId":1464,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1462,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1463,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1460,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1466,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1459,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1469,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1458,"metricType":"sum"},{"name":"data size","accumulatorId":1456,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1472,"metricType":"size"},{"name":"number of partitions","accumulatorId":1457,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1473,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1461,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1475,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1452,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1453,"metricType":"timing"},{"name":"peak memory","accumulatorId":1451,"metricType":"size"},{"name":"number of output rows","accumulatorId":1450,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1455,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1454,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1446,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1447,"metricType":"timing"},{"name":"peak memory","accumulatorId":1445,"metricType":"size"},{"name":"number of output rows","accumulatorId":1444,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1449,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1448,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1442,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1436,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1443,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1437,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1434,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1431,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1440,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1433,"metricType":"sum"},{"name":"records read","accumulatorId":1430,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1428,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1429,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1426,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1432,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1425,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1435,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1424,"metricType":"sum"},{"name":"data size","accumulatorId":1422,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1438,"metricType":"size"},{"name":"number of partitions","accumulatorId":1423,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1439,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1427,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1441,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1418,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1419,"metricType":"timing"},{"name":"peak memory","accumulatorId":1417,"metricType":"size"},{"name":"number of output rows","accumulatorId":1416,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1421,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1420,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1413,"metricType":"timing"},{"name":"peak memory","accumulatorId":1414,"metricType":"size"},{"name":"spill size","accumulatorId":1415,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1385,"metricType":"sum"},{"name":"spill size","accumulatorId":1386,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#37], [lesson_id#52], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#37 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1559,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1553,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1560,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1554,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1551,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1548,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1557,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1550,"metricType":"sum"},{"name":"records read","accumulatorId":1547,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1545,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1546,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1543,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1549,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1542,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1552,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1541,"metricType":"sum"},{"name":"data size","accumulatorId":1539,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1555,"metricType":"size"},{"name":"number of partitions","accumulatorId":1540,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1556,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1544,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1558,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1536,"metricType":"timing"},{"name":"peak memory","accumulatorId":1537,"metricType":"size"},{"name":"spill size","accumulatorId":1538,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#52 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1585,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1579,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1586,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1580,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1577,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1574,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1583,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1576,"metricType":"sum"},{"name":"records read","accumulatorId":1573,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1571,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1572,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1569,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1575,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1568,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1578,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1567,"metricType":"sum"},{"name":"data size","accumulatorId":1565,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1581,"metricType":"size"},{"name":"number of partitions","accumulatorId":1566,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1582,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1570,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1584,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1562,"metricType":"timing"},{"name":"peak memory","accumulatorId":1563,"metricType":"size"},{"name":"spill size","accumulatorId":1564,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1534,"metricType":"sum"},{"name":"spill size","accumulatorId":1535,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1530,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1531,"metricType":"timing"},{"name":"peak memory","accumulatorId":1529,"metricType":"size"},{"name":"number of output rows","accumulatorId":1528,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1533,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1532,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1524,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1525,"metricType":"timing"},{"name":"peak memory","accumulatorId":1523,"metricType":"size"},{"name":"number of output rows","accumulatorId":1522,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1527,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1526,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1518,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1519,"metricType":"timing"},{"name":"peak memory","accumulatorId":1517,"metricType":"size"},{"name":"number of output rows","accumulatorId":1516,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1521,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1520,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1514,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1508,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1515,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1509,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1506,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1503,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1512,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1505,"metricType":"sum"},{"name":"records read","accumulatorId":1502,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1500,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1501,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1498,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1504,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1497,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1507,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1496,"metricType":"sum"},{"name":"data size","accumulatorId":1494,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1510,"metricType":"size"},{"name":"number of partitions","accumulatorId":1495,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1511,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1499,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1513,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1490,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1491,"metricType":"timing"},{"name":"peak memory","accumulatorId":1489,"metricType":"size"},{"name":"number of output rows","accumulatorId":1488,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1493,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1492,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1485,"metricType":"timing"},{"name":"peak memory","accumulatorId":1486,"metricType":"size"},{"name":"spill size","accumulatorId":1487,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1383,"metricType":"sum"},{"name":"spill size","accumulatorId":1384,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1381,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1375,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1382,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1376,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1373,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1370,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1379,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1372,"metricType":"sum"},{"name":"records read","accumulatorId":1369,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1367,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1368,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1365,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1371,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1364,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1374,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1363,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1377,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1378,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1366,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1380,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784293429265,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (62)\n+- == Current Plan ==\n   CollectLimit (36)\n   +- Project (35)\n      +- SortMergeJoin Inner (34)\n         :- Project (16)\n         :  +- SortMergeJoin Inner (15)\n         :     :- Sort (5)\n         :     :  +- ShuffleQueryStage (4)\n         :     :     +- Exchange (3)\n         :     :        +- * Project (2)\n         :     :           +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (14)\n         :        +- HashAggregate (13)\n         :           +- Exchange (12)\n         :              +- HashAggregate (11)\n         :                 +- HashAggregate (10)\n         :                    +- ShuffleQueryStage (9)\n         :                       +- Exchange (8)\n         :                          +- * HashAggregate (7)\n         :                             +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         +- Sort (33)\n            +- HashAggregate (32)\n               +- Exchange (31)\n                  +- HashAggregate (30)\n                     +- HashAggregate (29)\n                        +- HashAggregate (28)\n                           +- Project (27)\n                              +- SortMergeJoin LeftOuter (26)\n                                 :- Sort (21)\n                                 :  +- ShuffleQueryStage (20)\n                                 :     +- Exchange (19)\n                                 :        +- * Project (18)\n                                 :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (17)\n                                 +- Sort (25)\n                                    +- ShuffleQueryStage (24)\n                                       +- Exchange (23)\n                                          +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n+- == Initial Plan ==\n   CollectLimit (61)\n   +- Project (60)\n      +- SortMergeJoin Inner (59)\n         :- Project (46)\n         :  +- SortMergeJoin Inner (45)\n         :     :- Sort (38)\n         :     :  +- Exchange (37)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (44)\n         :        +- HashAggregate (43)\n         :           +- Exchange (42)\n         :              +- HashAggregate (41)\n         :                 +- HashAggregate (40)\n         :                    +- Exchange (39)\n         :                       +- HashAggregate (7)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         +- Sort (58)\n            +- HashAggregate (57)\n               +- Exchange (56)\n                  +- HashAggregate (55)\n                     +- HashAggregate (54)\n                        +- HashAggregate (53)\n                           +- Project (52)\n                              +- SortMergeJoin LeftOuter (51)\n                                 :- Sort (48)\n                                 :  +- Exchange (47)\n                                 :     +- Project (18)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (17)\n                                 +- Sort (50)\n                                    +- Exchange (49)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(7) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(8) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(9) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(10) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1074]\n\n(13) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(14) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(15) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(16) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(17) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(18) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(19) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(20) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(21) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(23) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(24) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(25) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(26) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(28) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(29) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(30) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(31) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1122]\n\n(32) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(33) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(34) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(35) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(36) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(37) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(38) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(39) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(40) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(41) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(42) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(43) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(44) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(45) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(46) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(47) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(48) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(49) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(50) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(51) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(52) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(53) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(54) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(55) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(56) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(57) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(58) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(60) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(61) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(62) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#30], Inner","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#44], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#24 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1612,"metricType":"timing"},{"name":"peak memory","accumulatorId":1613,"metricType":"size"},{"name":"spill size","accumulatorId":1614,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#44 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1074]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1677,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1678,"metricType":"timing"},{"name":"peak memory","accumulatorId":1676,"metricType":"size"},{"name":"number of output rows","accumulatorId":1675,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1680,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1679,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1671,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1672,"metricType":"timing"},{"name":"peak memory","accumulatorId":1670,"metricType":"size"},{"name":"number of output rows","accumulatorId":1669,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1674,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1673,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1667,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1661,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1668,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1662,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1659,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1656,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1665,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1658,"metricType":"sum"},{"name":"records read","accumulatorId":1655,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1653,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1654,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1651,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1657,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1650,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1660,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1649,"metricType":"sum"},{"name":"data size","accumulatorId":1647,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1663,"metricType":"size"},{"name":"number of partitions","accumulatorId":1648,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1664,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1652,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1666,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1643,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1644,"metricType":"timing"},{"name":"peak memory","accumulatorId":1642,"metricType":"size"},{"name":"number of output rows","accumulatorId":1641,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1646,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1645,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1638,"metricType":"timing"},{"name":"peak memory","accumulatorId":1639,"metricType":"size"},{"name":"spill size","accumulatorId":1640,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"},{"name":"spill size","accumulatorId":1611,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1122]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#37], [lesson_id#52], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#37 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1755,"metricType":"timing"},{"name":"peak memory","accumulatorId":1756,"metricType":"size"},{"name":"spill size","accumulatorId":1757,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#52 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1781,"metricType":"timing"},{"name":"peak memory","accumulatorId":1782,"metricType":"size"},{"name":"spill size","accumulatorId":1783,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1753,"metricType":"sum"},{"name":"spill size","accumulatorId":1754,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1749,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1750,"metricType":"timing"},{"name":"peak memory","accumulatorId":1748,"metricType":"size"},{"name":"number of output rows","accumulatorId":1747,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1752,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1751,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1743,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1744,"metricType":"timing"},{"name":"peak memory","accumulatorId":1742,"metricType":"size"},{"name":"number of output rows","accumulatorId":1741,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1746,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1745,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1737,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1738,"metricType":"timing"},{"name":"peak memory","accumulatorId":1736,"metricType":"size"},{"name":"number of output rows","accumulatorId":1735,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1740,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1739,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1733,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1727,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1734,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1728,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1725,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1722,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1731,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1724,"metricType":"sum"},{"name":"records read","accumulatorId":1721,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1719,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1720,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1717,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1723,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1716,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1726,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1715,"metricType":"sum"},{"name":"data size","accumulatorId":1713,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1729,"metricType":"size"},{"name":"number of partitions","accumulatorId":1714,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1730,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1718,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1732,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1709,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1710,"metricType":"timing"},{"name":"peak memory","accumulatorId":1708,"metricType":"size"},{"name":"number of output rows","accumulatorId":1707,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1712,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1711,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1704,"metricType":"timing"},{"name":"peak memory","accumulatorId":1705,"metricType":"size"},{"name":"spill size","accumulatorId":1706,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1608,"metricType":"sum"},{"name":"spill size","accumulatorId":1609,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1606,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1600,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1607,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1601,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1598,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1595,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1604,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1597,"metricType":"sum"},{"name":"records read","accumulatorId":1594,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1592,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1593,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1590,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1596,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1589,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1599,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1588,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1602,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1603,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1591,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1605,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1616,200]]}
{"Event":"SparkListenerJobStart","Job ID":12,"Submission Time":1784293429281,"Stage Infos":[{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[26],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429281,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":26,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429284,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1682,200]]}
{"Event":"SparkListenerJobStart","Job ID":13,"Submission Time":1784293429299,"Stage Infos":[{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"130\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[27],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"130\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429300,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":27,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429304,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1759,200]]}
{"Event":"SparkListenerJobStart","Job ID":14,"Submission Time":1784293429306,"Stage Infos":[{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"134\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[28],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"134\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429306,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1785,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":28,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429312,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":15,"Submission Time":1784293429313,"Stage Infos":[{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"138\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[29],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"138\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429314,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":29,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429325,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":27,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429304,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429397,"Failed":false,"Killed":false,"Accumulables":[{"ID":1478,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1479,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1481,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1484,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"data size","Update":"1080","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle bytes written","Update":"2676","Value":"2676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1701,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"shuffle write time","Update":"4144967","Value":"4144967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1703,"Name":"duration","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1842,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7016249,"Value":7016249,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.executorRunTime","Update":71,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.executorCpuTime","Update":43146225,"Value":43146225,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1851,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1870,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2676,"Value":2676,"Internal":true,"Count Failed Values":true},{"ID":1871,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1872,"Name":"internal.metrics.shuffle.write.writeTime","Update":4144967,"Value":4144967,"Internal":true,"Count Failed Values":true},{"ID":1874,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":7016249,"Executor Run Time":71,"Executor CPU Time":43146225,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2676,"Shuffle Write Time":4144967,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"130\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429300,"Completion Time":1784293429398,"Accumulables":[{"ID":1478,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1479,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1481,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1484,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1681,"Name":"data size","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1700,"Name":"shuffle bytes written","Value":"2676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1701,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1702,"Name":"shuffle write time","Value":"4144967","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1703,"Name":"duration","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1842,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":1843,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7016249,"Internal":true,"Count Failed Values":true},{"ID":1844,"Name":"internal.metrics.executorRunTime","Value":71,"Internal":true,"Count Failed Values":true},{"ID":1845,"Name":"internal.metrics.executorCpuTime","Value":43146225,"Internal":true,"Count Failed Values":true},{"ID":1846,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1851,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1870,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2676,"Internal":true,"Count Failed Values":true},{"ID":1871,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":1872,"Name":"internal.metrics.shuffle.write.writeTime","Value":4144967,"Internal":true,"Count Failed Values":true},{"ID":1874,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":13,"Completion Time":1784293429401,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":28,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429312,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429400,"Failed":false,"Killed":false,"Accumulables":[{"ID":1561,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1758,"Name":"data size","Update":"432","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1777,"Name":"shuffle bytes written","Update":"1031","Value":"1031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1778,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1779,"Name":"shuffle write time","Update":"4444201","Value":"4444201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1780,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1877,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1878,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5298984,"Value":5298984,"Internal":true,"Count Failed Values":true},{"ID":1879,"Name":"internal.metrics.executorRunTime","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":1880,"Name":"internal.metrics.executorCpuTime","Update":17935291,"Value":17935291,"Internal":true,"Count Failed Values":true},{"ID":1881,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1905,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1031,"Value":1031,"Internal":true,"Count Failed Values":true},{"ID":1906,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":1907,"Name":"internal.metrics.shuffle.write.writeTime","Update":4444201,"Value":4444201,"Internal":true,"Count Failed Values":true},{"ID":1909,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":5298984,"Executor Run Time":54,"Executor CPU Time":17935291,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1031,"Shuffle Write Time":4444201,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"134\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429306,"Completion Time":1784293429406,"Accumulables":[{"ID":1561,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1758,"Name":"data size","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1777,"Name":"shuffle bytes written","Value":"1031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1778,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1779,"Name":"shuffle write time","Value":"4444201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1780,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1877,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1878,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5298984,"Internal":true,"Count Failed Values":true},{"ID":1879,"Name":"internal.metrics.executorRunTime","Value":54,"Internal":true,"Count Failed Values":true},{"ID":1880,"Name":"internal.metrics.executorCpuTime","Value":17935291,"Internal":true,"Count Failed Values":true},{"ID":1881,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1905,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1031,"Internal":true,"Count Failed Values":true},{"ID":1906,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":1907,"Name":"internal.metrics.shuffle.write.writeTime","Value":4444201,"Internal":true,"Count Failed Values":true},{"ID":1909,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":14,"Completion Time":1784293429408,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":29,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429325,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429406,"Failed":false,"Killed":false,"Accumulables":[{"ID":1587,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1784,"Name":"data size","Update":"1368","Value":"1368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"shuffle bytes written","Update":"1298","Value":"1298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"shuffle records written","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1805,"Name":"shuffle write time","Update":"4546856","Value":"4546856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1806,"Name":"duration","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1912,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1913,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5097503,"Value":5097503,"Internal":true,"Count Failed Values":true},{"ID":1914,"Name":"internal.metrics.executorRunTime","Update":56,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":1915,"Name":"internal.metrics.executorCpuTime","Update":18595923,"Value":18595923,"Internal":true,"Count Failed Values":true},{"ID":1916,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1940,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1298,"Value":1298,"Internal":true,"Count Failed Values":true},{"ID":1941,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":1942,"Name":"internal.metrics.shuffle.write.writeTime","Update":4546856,"Value":4546856,"Internal":true,"Count Failed Values":true},{"ID":1944,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":5097503,"Executor Run Time":56,"Executor CPU Time":18595923,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1298,"Shuffle Write Time":4546856,"Shuffle Records Written":57},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"138\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429314,"Completion Time":1784293429409,"Accumulables":[{"ID":1587,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1784,"Name":"data size","Value":"1368","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1803,"Name":"shuffle bytes written","Value":"1298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1804,"Name":"shuffle records written","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1805,"Name":"shuffle write time","Value":"4546856","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1806,"Name":"duration","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1912,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":1913,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5097503,"Internal":true,"Count Failed Values":true},{"ID":1914,"Name":"internal.metrics.executorRunTime","Value":56,"Internal":true,"Count Failed Values":true},{"ID":1915,"Name":"internal.metrics.executorCpuTime","Value":18595923,"Internal":true,"Count Failed Values":true},{"ID":1916,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1940,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1298,"Internal":true,"Count Failed Values":true},{"ID":1941,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":57,"Internal":true,"Count Failed Values":true},{"ID":1942,"Name":"internal.metrics.shuffle.write.writeTime","Value":4546856,"Internal":true,"Count Failed Values":true},{"ID":1944,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":15,"Completion Time":1784293429412,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (64)\n+- == Current Plan ==\n   CollectLimit (38)\n   +- Project (37)\n      +- SortMergeJoin Inner (36)\n         :- Sort (18)\n         :  +- Project (17)\n         :     +- BroadcastHashJoin Inner BuildRight (16)\n         :        :- ShuffleQueryStage (4)\n         :        :  +- Exchange (3)\n         :        :     +- * Project (2)\n         :        :        +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :        +- BroadcastExchange (15)\n         :           +- HashAggregate (14)\n         :              +- ShuffleQueryStage (13)\n         :                 +- Exchange (12)\n         :                    +- * HashAggregate (11)\n         :                       +- * HashAggregate (10)\n         :                          +- AQEShuffleRead (9)\n         :                             +- ShuffleQueryStage (8), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                                +- Exchange (7)\n         :                                   +- * HashAggregate (6)\n         :                                      +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (35)\n            +- HashAggregate (34)\n               +- Exchange (33)\n                  +- HashAggregate (32)\n                     +- HashAggregate (31)\n                        +- HashAggregate (30)\n                           +- Project (29)\n                              +- SortMergeJoin LeftOuter (28)\n                                 :- Sort (23)\n                                 :  +- ShuffleQueryStage (22)\n                                 :     +- Exchange (21)\n                                 :        +- * Project (20)\n                                 :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (19)\n                                 +- Sort (27)\n                                    +- ShuffleQueryStage (26)\n                                       +- Exchange (25)\n                                          +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (24)\n+- == Initial Plan ==\n   CollectLimit (63)\n   +- Project (62)\n      +- SortMergeJoin Inner (61)\n         :- Project (48)\n         :  +- SortMergeJoin Inner (47)\n         :     :- Sort (40)\n         :     :  +- Exchange (39)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (46)\n         :        +- HashAggregate (45)\n         :           +- Exchange (44)\n         :              +- HashAggregate (43)\n         :                 +- HashAggregate (42)\n         :                    +- Exchange (41)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (60)\n            +- HashAggregate (59)\n               +- Exchange (58)\n                  +- HashAggregate (57)\n                     +- HashAggregate (56)\n                        +- HashAggregate (55)\n                           +- Project (54)\n                              +- SortMergeJoin LeftOuter (53)\n                                 :- Sort (50)\n                                 :  +- Exchange (49)\n                                 :     +- Project (20)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (19)\n                                 +- Sort (52)\n                                    +- Exchange (51)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (24)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(13) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(14) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(15) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1214]\n\n(16) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(17) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(18) Sort\nInput [3]: [course_id#24, title#17, unique_users#150L]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(19) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(20) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(21) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(22) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(23) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(24) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(25) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(26) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(27) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(28) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(29) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(30) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(31) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(32) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(33) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1176]\n\n(34) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(35) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(37) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(38) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(39) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(40) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(41) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(42) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(43) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(44) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(45) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(46) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(47) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(48) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(49) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(50) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(51) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(52) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(54) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(55) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(56) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(57) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(58) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(59) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(60) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(61) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(62) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(63) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(64) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#30], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#24 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1214]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1980,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1981,"metricType":"timing"},{"name":"peak memory","accumulatorId":1979,"metricType":"size"},{"name":"number of output rows","accumulatorId":1978,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1983,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1982,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1977,"metricType":"timing"},{"name":"time to build","accumulatorId":1976,"metricType":"timing"},{"name":"time to collect","accumulatorId":1975,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1974,"metricType":"sum"},{"name":"data size","accumulatorId":1973,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1972,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1969,"metricType":"timing"},{"name":"peak memory","accumulatorId":1970,"metricType":"size"},{"name":"spill size","accumulatorId":1971,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1176]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#37], [lesson_id#52], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#37 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2073,"metricType":"timing"},{"name":"peak memory","accumulatorId":2074,"metricType":"size"},{"name":"spill size","accumulatorId":2075,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#52 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2076,"metricType":"timing"},{"name":"peak memory","accumulatorId":2077,"metricType":"size"},{"name":"spill size","accumulatorId":2078,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2071,"metricType":"sum"},{"name":"spill size","accumulatorId":2072,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2067,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2068,"metricType":"timing"},{"name":"peak memory","accumulatorId":2066,"metricType":"size"},{"name":"number of output rows","accumulatorId":2065,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2070,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2069,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2061,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2062,"metricType":"timing"},{"name":"peak memory","accumulatorId":2060,"metricType":"size"},{"name":"number of output rows","accumulatorId":2059,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2064,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2063,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2055,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2056,"metricType":"timing"},{"name":"peak memory","accumulatorId":2054,"metricType":"size"},{"name":"number of output rows","accumulatorId":2053,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2058,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2057,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2051,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2045,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2052,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2046,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2043,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2040,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2049,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2042,"metricType":"sum"},{"name":"records read","accumulatorId":2039,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2037,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2038,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2035,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2041,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2034,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2044,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2033,"metricType":"sum"},{"name":"data size","accumulatorId":2031,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2047,"metricType":"size"},{"name":"number of partitions","accumulatorId":2032,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2048,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2036,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2050,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2027,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2028,"metricType":"timing"},{"name":"peak memory","accumulatorId":2026,"metricType":"size"},{"name":"number of output rows","accumulatorId":2025,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2030,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2029,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2022,"metricType":"timing"},{"name":"peak memory","accumulatorId":2023,"metricType":"size"},{"name":"spill size","accumulatorId":2024,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1967,"metricType":"sum"},{"name":"spill size","accumulatorId":1968,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2019,1],[2021,1],[2020,2802]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1985,200]]}
{"Event":"SparkListenerJobStart","Job ID":16,"Submission Time":1784293429441,"Stage Infos":[{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"130\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"145\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[30,31],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"145\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429443,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":31,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429447,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (64)\n+- == Current Plan ==\n   CollectLimit (38)\n   +- Project (37)\n      +- BroadcastHashJoin Inner BuildRight (36)\n         :- Project (17)\n         :  +- BroadcastHashJoin Inner BuildRight (16)\n         :     :- ShuffleQueryStage (4)\n         :     :  +- Exchange (3)\n         :     :     +- * Project (2)\n         :     :        +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- BroadcastExchange (15)\n         :        +- HashAggregate (14)\n         :           +- ShuffleQueryStage (13)\n         :              +- Exchange (12)\n         :                 +- * HashAggregate (11)\n         :                    +- * HashAggregate (10)\n         :                       +- AQEShuffleRead (9)\n         :                          +- ShuffleQueryStage (8), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                             +- Exchange (7)\n         :                                +- * HashAggregate (6)\n         :                                   +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- BroadcastExchange (35)\n            +- HashAggregate (34)\n               +- Exchange (33)\n                  +- HashAggregate (32)\n                     +- HashAggregate (31)\n                        +- HashAggregate (30)\n                           +- Project (29)\n                              +- BroadcastHashJoin LeftOuter BuildRight (28)\n                                 :- ShuffleQueryStage (21), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                 :  +- Exchange (20)\n                                 :     +- * Project (19)\n                                 :        +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (18)\n                                 +- BroadcastQueryStage (27)\n                                    +- BroadcastExchange (26)\n                                       +- AQEShuffleRead (25)\n                                          +- ShuffleQueryStage (24), Statistics(sizeInBytes=1368.0 B, rowCount=57)\n                                             +- Exchange (23)\n                                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n+- == Initial Plan ==\n   CollectLimit (63)\n   +- Project (62)\n      +- SortMergeJoin Inner (61)\n         :- Project (48)\n         :  +- SortMergeJoin Inner (47)\n         :     :- Sort (40)\n         :     :  +- Exchange (39)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (46)\n         :        +- HashAggregate (45)\n         :           +- Exchange (44)\n         :              +- HashAggregate (43)\n         :                 +- HashAggregate (42)\n         :                    +- Exchange (41)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (60)\n            +- HashAggregate (59)\n               +- Exchange (58)\n                  +- HashAggregate (57)\n                     +- HashAggregate (56)\n                        +- HashAggregate (55)\n                           +- Project (54)\n                              +- SortMergeJoin LeftOuter (53)\n                                 :- Sort (50)\n                                 :  +- Exchange (49)\n                                 :     +- Project (19)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (18)\n                                 +- Sort (52)\n                                    +- Exchange (51)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(13) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(14) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(15) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1273]\n\n(16) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(17) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(18) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(19) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(20) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(21) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(23) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(24) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: local\n\n(26) BroadcastExchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]\n\n(27) BroadcastQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 5\n\n(28) BroadcastHashJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(29) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(30) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(31) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(32) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(33) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1320]\n\n(34) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(35) BroadcastExchange\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1322]\n\n(36) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(37) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(38) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(39) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(40) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(41) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(42) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(43) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(44) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(45) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(46) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(47) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(48) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(49) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(50) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(51) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(52) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(54) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(55) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(56) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(57) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(58) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(59) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(60) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(61) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(62) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(63) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(64) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#30], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1273]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1980,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1981,"metricType":"timing"},{"name":"peak memory","accumulatorId":1979,"metricType":"size"},{"name":"number of output rows","accumulatorId":1978,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1983,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1982,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2140,"metricType":"timing"},{"name":"time to build","accumulatorId":2139,"metricType":"timing"},{"name":"time to collect","accumulatorId":2138,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2137,"metricType":"sum"},{"name":"data size","accumulatorId":2136,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2135,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1322]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1320]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#37], [lesson_id#52], LeftOuter, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2198,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2197,"metricType":"timing"},{"name":"time to build","accumulatorId":2196,"metricType":"timing"},{"name":"time to collect","accumulatorId":2195,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2194,"metricType":"sum"},{"name":"data size","accumulatorId":2193,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2192,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2189,"metricType":"timing"},{"name":"peak memory","accumulatorId":2187,"metricType":"size"},{"name":"number of output rows","accumulatorId":2186,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2191,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2182,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2183,"metricType":"timing"},{"name":"peak memory","accumulatorId":2181,"metricType":"size"},{"name":"number of output rows","accumulatorId":2180,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2185,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2184,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2176,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2177,"metricType":"timing"},{"name":"peak memory","accumulatorId":2175,"metricType":"size"},{"name":"number of output rows","accumulatorId":2174,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2179,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2178,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2172,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2166,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2173,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2167,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2164,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2161,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2170,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2163,"metricType":"sum"},{"name":"records read","accumulatorId":2160,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2158,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2159,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2156,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2162,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2155,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2165,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2154,"metricType":"sum"},{"name":"data size","accumulatorId":2152,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2168,"metricType":"size"},{"name":"number of partitions","accumulatorId":2153,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2169,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2157,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2171,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2148,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2149,"metricType":"timing"},{"name":"peak memory","accumulatorId":2147,"metricType":"size"},{"name":"number of output rows","accumulatorId":2146,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2151,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2150,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2145,"metricType":"timing"},{"name":"time to build","accumulatorId":2144,"metricType":"timing"},{"name":"time to collect","accumulatorId":2143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2142,"metricType":"sum"},{"name":"data size","accumulatorId":2141,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2134,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2132,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2126,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2133,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2127,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2124,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2121,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2130,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2123,"metricType":"sum"},{"name":"records read","accumulatorId":2120,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2118,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2119,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2116,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2122,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2115,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2125,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2114,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2128,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2129,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2117,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2131,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2198,1]]}
{"Event":"SparkListenerJobStart","Job ID":17,"Submission Time":1784293429460,"Stage Infos":[{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"147\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"138\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[33,32],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"146\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f64e81f7-1fe8-488e-9ec4-afe882da0564)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"147\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429460,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"146\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f64e81f7-1fe8-488e-9ec4-afe882da0564)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429464,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429464,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429492,"Failed":false,"Killed":false,"Accumulables":[{"ID":1787,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1790,"Name":"local bytes read","Update":"1298","Value":"1298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1791,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1792,"Name":"records read","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"internal.metrics.executorDeserializeTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":2200,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4798418,"Value":4798418,"Internal":true,"Count Failed Values":true},{"ID":2201,"Name":"internal.metrics.executorRunTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2202,"Name":"internal.metrics.executorCpuTime","Update":4275088,"Value":4275088,"Internal":true,"Count Failed Values":true},{"ID":2203,"Name":"internal.metrics.resultSize","Update":4351,"Value":4351,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2214,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1298,"Value":1298,"Internal":true,"Count Failed Values":true},{"ID":2215,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2216,"Name":"internal.metrics.shuffle.read.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2218,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2219,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2220,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2221,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2222,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2223,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2224,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2225,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2226,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":9,"Executor Deserialize CPU Time":4798418,"Executor Run Time":11,"Executor CPU Time":4275088,"Peak Execution Memory":0,"Result Size":4351,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1298,"Total Records Read":57,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"147\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429460,"Completion Time":1784293429494,"Accumulables":[{"ID":1787,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1790,"Name":"local bytes read","Value":"1298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1791,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1792,"Name":"records read","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2199,"Name":"internal.metrics.executorDeserializeTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":2200,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4798418,"Internal":true,"Count Failed Values":true},{"ID":2201,"Name":"internal.metrics.executorRunTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":2202,"Name":"internal.metrics.executorCpuTime","Value":4275088,"Internal":true,"Count Failed Values":true},{"ID":2203,"Name":"internal.metrics.resultSize","Value":4351,"Internal":true,"Count Failed Values":true},{"ID":2210,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2211,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2212,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2213,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2214,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1298,"Internal":true,"Count Failed Values":true},{"ID":2215,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2216,"Name":"internal.metrics.shuffle.read.recordsRead","Value":57,"Internal":true,"Count Failed Values":true},{"ID":2217,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2218,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2219,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2220,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2221,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2222,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2223,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2224,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2225,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2226,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":17,"Completion Time":1784293429494,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2197,2],[2196,2],[2195,41],[2194,57],[2193,1048704]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (67)\n+- == Current Plan ==\n   CollectLimit (41)\n   +- Project (40)\n      +- SortMergeJoin Inner (39)\n         :- Sort (18)\n         :  +- Project (17)\n         :     +- BroadcastHashJoin Inner BuildRight (16)\n         :        :- ShuffleQueryStage (4)\n         :        :  +- Exchange (3)\n         :        :     +- * Project (2)\n         :        :        +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :        +- BroadcastExchange (15)\n         :           +- HashAggregate (14)\n         :              +- ShuffleQueryStage (13)\n         :                 +- Exchange (12)\n         :                    +- * HashAggregate (11)\n         :                       +- * HashAggregate (10)\n         :                          +- AQEShuffleRead (9)\n         :                             +- ShuffleQueryStage (8), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                                +- Exchange (7)\n         :                                   +- * HashAggregate (6)\n         :                                      +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (38)\n            +- HashAggregate (37)\n               +- ShuffleQueryStage (36)\n                  +- Exchange (35)\n                     +- * HashAggregate (34)\n                        +- * HashAggregate (33)\n                           +- * HashAggregate (32)\n                              +- * Project (31)\n                                 +- * BroadcastHashJoin LeftOuter BuildRight (30)\n                                    :- AQEShuffleRead (23)\n                                    :  +- ShuffleQueryStage (22), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                    :     +- Exchange (21)\n                                    :        +- * Project (20)\n                                    :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (19)\n                                    +- BroadcastQueryStage (29), Statistics(sizeInBytes=1024.1 KiB, rowCount=57)\n                                       +- BroadcastExchange (28)\n                                          +- AQEShuffleRead (27)\n                                             +- ShuffleQueryStage (26), Statistics(sizeInBytes=1368.0 B, rowCount=57)\n                                                +- Exchange (25)\n                                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (24)\n+- == Initial Plan ==\n   CollectLimit (66)\n   +- Project (65)\n      +- SortMergeJoin Inner (64)\n         :- Project (51)\n         :  +- SortMergeJoin Inner (50)\n         :     :- Sort (43)\n         :     :  +- Exchange (42)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (49)\n         :        +- HashAggregate (48)\n         :           +- Exchange (47)\n         :              +- HashAggregate (46)\n         :                 +- HashAggregate (45)\n         :                    +- Exchange (44)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (63)\n            +- HashAggregate (62)\n               +- Exchange (61)\n                  +- HashAggregate (60)\n                     +- HashAggregate (59)\n                        +- HashAggregate (58)\n                           +- Project (57)\n                              +- SortMergeJoin LeftOuter (56)\n                                 :- Sort (53)\n                                 :  +- Exchange (52)\n                                 :     +- Project (20)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (19)\n                                 +- Sort (55)\n                                    +- Exchange (54)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (24)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(13) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(14) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(15) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1362]\n\n(16) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(17) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(18) Sort\nInput [3]: [course_id#24, title#17, unique_users#150L]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(19) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(20) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(21) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(22) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(23) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(24) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(25) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(26) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(27) AQEShuffleRead\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: local\n\n(28) BroadcastExchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]\n\n(29) BroadcastQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 5\n\n(30) BroadcastHashJoin [codegen id : 6]\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(31) Project [codegen id : 6]\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(32) HashAggregate [codegen id : 6]\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(33) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(34) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(35) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]\n\n(36) ShuffleQueryStage\nOutput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: 6\n\n(37) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(38) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(39) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(40) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(41) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(42) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(43) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(44) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(45) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(46) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(47) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(48) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(49) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(52) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(53) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(55) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(57) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(58) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(59) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(60) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(61) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(62) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(63) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(64) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(65) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(66) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(67) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#30], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#24 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1362]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1980,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1981,"metricType":"timing"},{"name":"peak memory","accumulatorId":1979,"metricType":"size"},{"name":"number of output rows","accumulatorId":1978,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1983,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1982,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2264,"metricType":"timing"},{"name":"time to build","accumulatorId":2263,"metricType":"timing"},{"name":"time to collect","accumulatorId":2262,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2261,"metricType":"sum"},{"name":"data size","accumulatorId":2260,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2259,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2256,"metricType":"timing"},{"name":"peak memory","accumulatorId":2257,"metricType":"size"},{"name":"spill size","accumulatorId":2258,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#37], [lesson_id#52], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2316,"metricType":"sum"},{"name":"partition data size","accumulatorId":2317,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2318,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2198,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2197,"metricType":"timing"},{"name":"time to build","accumulatorId":2196,"metricType":"timing"},{"name":"time to collect","accumulatorId":2195,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2194,"metricType":"sum"},{"name":"data size","accumulatorId":2193,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2315,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2311,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2312,"metricType":"timing"},{"name":"peak memory","accumulatorId":2310,"metricType":"size"},{"name":"number of output rows","accumulatorId":2309,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2314,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2313,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2305,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2306,"metricType":"timing"},{"name":"peak memory","accumulatorId":2304,"metricType":"size"},{"name":"number of output rows","accumulatorId":2303,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2308,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2307,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2299,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2300,"metricType":"timing"},{"name":"peak memory","accumulatorId":2298,"metricType":"size"},{"name":"number of output rows","accumulatorId":2297,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2302,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2301,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2296,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2294,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2295,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2289,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2286,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2283,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2292,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2285,"metricType":"sum"},{"name":"records read","accumulatorId":2282,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2280,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2281,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2278,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2284,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2277,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2276,"metricType":"sum"},{"name":"data size","accumulatorId":2274,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2290,"metricType":"size"},{"name":"number of partitions","accumulatorId":2275,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2291,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2279,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2293,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2270,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2271,"metricType":"timing"},{"name":"peak memory","accumulatorId":2269,"metricType":"size"},{"name":"number of output rows","accumulatorId":2268,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2273,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2272,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2265,"metricType":"timing"},{"name":"peak memory","accumulatorId":2266,"metricType":"size"},{"name":"spill size","accumulatorId":2267,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2254,"metricType":"sum"},{"name":"spill size","accumulatorId":2255,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2252,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2246,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2253,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2247,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2244,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2241,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2250,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2243,"metricType":"sum"},{"name":"records read","accumulatorId":2240,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2238,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2239,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2236,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2242,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2235,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2245,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2234,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2248,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2249,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2237,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2251,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":31,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429447,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429548,"Failed":false,"Killed":false,"Accumulables":[{"ID":1684,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"local bytes read","Update":"2676","Value":"2676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1689,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"data size","Update":"120","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2003,"Name":"shuffle bytes written","Update":"315","Value":"315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2004,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2005,"Name":"shuffle write time","Update":"2769756","Value":"2769756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2006,"Name":"duration","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2007,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2008,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2010,"Name":"time in aggregation build","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2013,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2014,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2016,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2079,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2080,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7325897,"Value":7325897,"Internal":true,"Count Failed Values":true},{"ID":2081,"Name":"internal.metrics.executorRunTime","Update":81,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":2082,"Name":"internal.metrics.executorCpuTime","Update":58171778,"Value":58171778,"Internal":true,"Count Failed Values":true},{"ID":2083,"Name":"internal.metrics.resultSize","Update":5603,"Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2084,"Name":"internal.metrics.jvmGCTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2090,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2091,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2676,"Value":2676,"Internal":true,"Count Failed Values":true},{"ID":2095,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2096,"Name":"internal.metrics.shuffle.read.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2097,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2105,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":315,"Value":315,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.shuffle.write.writeTime","Update":2769756,"Value":2769756,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":7325897,"Executor Run Time":81,"Executor CPU Time":58171778,"Peak Execution Memory":524288,"Result Size":5603,"JVM GC Time":15,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2676,"Total Records Read":45,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":315,"Shuffle Write Time":2769756,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"145\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429443,"Completion Time":1784293429549,"Accumulables":[{"ID":1684,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1687,"Name":"local bytes read","Value":"2676","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1688,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1689,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"data size","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2003,"Name":"shuffle bytes written","Value":"315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2004,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2005,"Name":"shuffle write time","Value":"2769756","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2006,"Name":"duration","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2007,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2008,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2010,"Name":"time in aggregation build","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2013,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2014,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2016,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2079,"Name":"internal.metrics.executorDeserializeTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":2080,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7325897,"Internal":true,"Count Failed Values":true},{"ID":2081,"Name":"internal.metrics.executorRunTime","Value":81,"Internal":true,"Count Failed Values":true},{"ID":2082,"Name":"internal.metrics.executorCpuTime","Value":58171778,"Internal":true,"Count Failed Values":true},{"ID":2083,"Name":"internal.metrics.resultSize","Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2084,"Name":"internal.metrics.jvmGCTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":2088,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2090,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2091,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2092,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2093,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2094,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2676,"Internal":true,"Count Failed Values":true},{"ID":2095,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2096,"Name":"internal.metrics.shuffle.read.recordsRead","Value":45,"Internal":true,"Count Failed Values":true},{"ID":2097,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2101,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2103,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2104,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2105,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":315,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.shuffle.write.writeTime","Value":2769756,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":16,"Completion Time":1784293429550,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2316,1],[2318,1],[2317,1078]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2275,200]]}
{"Event":"SparkListenerJobStart","Job ID":18,"Submission Time":1784293429579,"Stage Infos":[{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"134\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"163\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"154\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[34,35],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"163\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"154\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429580,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429586,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (69)\n+- == Current Plan ==\n   CollectLimit (43)\n   +- Project (42)\n      +- SortMergeJoin Inner (41)\n         :- Sort (20)\n         :  +- Project (19)\n         :     +- BroadcastHashJoin Inner BuildRight (18)\n         :        :- ShuffleQueryStage (4)\n         :        :  +- Exchange (3)\n         :        :     +- * Project (2)\n         :        :        +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :        +- BroadcastQueryStage (17)\n         :           +- BroadcastExchange (16)\n         :              +- * HashAggregate (15)\n         :                 +- AQEShuffleRead (14)\n         :                    +- ShuffleQueryStage (13), Statistics(sizeInBytes=120.0 B, rowCount=5)\n         :                       +- Exchange (12)\n         :                          +- * HashAggregate (11)\n         :                             +- * HashAggregate (10)\n         :                                +- AQEShuffleRead (9)\n         :                                   +- ShuffleQueryStage (8), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                                      +- Exchange (7)\n         :                                         +- * HashAggregate (6)\n         :                                            +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (40)\n            +- HashAggregate (39)\n               +- ShuffleQueryStage (38)\n                  +- Exchange (37)\n                     +- * HashAggregate (36)\n                        +- * HashAggregate (35)\n                           +- * HashAggregate (34)\n                              +- * Project (33)\n                                 +- * BroadcastHashJoin LeftOuter BuildRight (32)\n                                    :- AQEShuffleRead (25)\n                                    :  +- ShuffleQueryStage (24), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                    :     +- Exchange (23)\n                                    :        +- * Project (22)\n                                    :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (21)\n                                    +- BroadcastQueryStage (31), Statistics(sizeInBytes=1024.1 KiB, rowCount=57)\n                                       +- BroadcastExchange (30)\n                                          +- AQEShuffleRead (29)\n                                             +- ShuffleQueryStage (28), Statistics(sizeInBytes=1368.0 B, rowCount=57)\n                                                +- Exchange (27)\n                                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (26)\n+- == Initial Plan ==\n   CollectLimit (68)\n   +- Project (67)\n      +- SortMergeJoin Inner (66)\n         :- Project (53)\n         :  +- SortMergeJoin Inner (52)\n         :     :- Sort (45)\n         :     :  +- Exchange (44)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (51)\n         :        +- HashAggregate (50)\n         :           +- Exchange (49)\n         :              +- HashAggregate (48)\n         :                 +- HashAggregate (47)\n         :                    +- Exchange (46)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (65)\n            +- HashAggregate (64)\n               +- Exchange (63)\n                  +- HashAggregate (62)\n                     +- HashAggregate (61)\n                        +- HashAggregate (60)\n                           +- Project (59)\n                              +- SortMergeJoin LeftOuter (58)\n                                 :- Sort (55)\n                                 :  +- Exchange (54)\n                                 :     +- Project (22)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (21)\n                                 +- Sort (57)\n                                    +- Exchange (56)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (26)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(13) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(14) AQEShuffleRead\nInput [2]: [course_id#44, count#399L]\nArguments: coalesced\n\n(15) HashAggregate [codegen id : 7]\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(16) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]\n\n(17) BroadcastQueryStage\nOutput [2]: [course_id#44, unique_users#150L]\nArguments: 7\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(20) Sort\nInput [3]: [course_id#24, title#17, unique_users#150L]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(21) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(22) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(23) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(24) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(25) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(26) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(27) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(28) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(29) AQEShuffleRead\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: local\n\n(30) BroadcastExchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]\n\n(31) BroadcastQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 5\n\n(32) BroadcastHashJoin [codegen id : 6]\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project [codegen id : 6]\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(34) HashAggregate [codegen id : 6]\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(35) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(36) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(37) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]\n\n(38) ShuffleQueryStage\nOutput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: 6\n\n(39) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(40) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(42) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(43) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(44) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(45) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(46) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(47) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(48) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(49) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(50) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(51) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(52) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(53) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(54) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(55) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(56) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(57) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(58) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(59) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(60) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(61) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(62) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(63) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(64) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(65) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(67) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(68) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(69) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#24], [course_id#30], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#24 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2392,"metricType":"sum"},{"name":"partition data size","accumulatorId":2393,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2394,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2388,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2389,"metricType":"timing"},{"name":"peak memory","accumulatorId":2387,"metricType":"size"},{"name":"number of output rows","accumulatorId":2386,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2391,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2390,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2385,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2384,"metricType":"timing"},{"name":"time to build","accumulatorId":2383,"metricType":"timing"},{"name":"time to collect","accumulatorId":2382,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2380,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2379,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2376,"metricType":"timing"},{"name":"peak memory","accumulatorId":2377,"metricType":"size"},{"name":"spill size","accumulatorId":2378,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#30 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#37], [lesson_id#52], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2316,"metricType":"sum"},{"name":"partition data size","accumulatorId":2317,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2318,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2198,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2197,"metricType":"timing"},{"name":"time to build","accumulatorId":2196,"metricType":"timing"},{"name":"time to collect","accumulatorId":2195,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2194,"metricType":"sum"},{"name":"data size","accumulatorId":2193,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2315,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2311,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2312,"metricType":"timing"},{"name":"peak memory","accumulatorId":2310,"metricType":"size"},{"name":"number of output rows","accumulatorId":2309,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2314,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2313,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2305,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2306,"metricType":"timing"},{"name":"peak memory","accumulatorId":2304,"metricType":"size"},{"name":"number of output rows","accumulatorId":2303,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2308,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2307,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2299,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2300,"metricType":"timing"},{"name":"peak memory","accumulatorId":2298,"metricType":"size"},{"name":"number of output rows","accumulatorId":2297,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2302,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2301,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2296,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2294,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2295,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2289,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2286,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2283,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2292,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2285,"metricType":"sum"},{"name":"records read","accumulatorId":2282,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2280,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2281,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2278,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2284,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2277,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2276,"metricType":"sum"},{"name":"data size","accumulatorId":2274,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2290,"metricType":"size"},{"name":"number of partitions","accumulatorId":2275,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2291,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2279,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2293,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2270,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2271,"metricType":"timing"},{"name":"peak memory","accumulatorId":2269,"metricType":"size"},{"name":"number of output rows","accumulatorId":2268,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2273,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2272,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2265,"metricType":"timing"},{"name":"peak memory","accumulatorId":2266,"metricType":"size"},{"name":"spill size","accumulatorId":2267,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2374,"metricType":"sum"},{"name":"spill size","accumulatorId":2375,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2372,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2366,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2373,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2367,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2364,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2361,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2370,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2363,"metricType":"sum"},{"name":"records read","accumulatorId":2360,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2358,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2359,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2356,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2362,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2355,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2365,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2354,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2368,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2369,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2357,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2371,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2392,1],[2394,1],[2393,330]]}
{"Event":"SparkListenerJobStart","Job ID":19,"Submission Time":1784293429607,"Stage Infos":[{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"140\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":52,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"145\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"164\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"168\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":36,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"130\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[37,38,36],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"164\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 76bee961-3794-459b-9b87-f66334c3a812)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"164\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"168\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429609,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"164\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 76bee961-3794-459b-9b87-f66334c3a812)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":38,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429620,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":38,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429620,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429660,"Failed":false,"Killed":false,"Accumulables":[{"ID":1987,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1990,"Name":"local bytes read","Update":"315","Value":"315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1991,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1992,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2387,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2389,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2390,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2395,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2396,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7587798,"Value":7587798,"Internal":true,"Count Failed Values":true},{"ID":2397,"Name":"internal.metrics.executorRunTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2398,"Name":"internal.metrics.executorCpuTime","Update":15817712,"Value":15817712,"Internal":true,"Count Failed Values":true},{"ID":2399,"Name":"internal.metrics.resultSize","Update":6897,"Value":6897,"Internal":true,"Count Failed Values":true},{"ID":2404,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2410,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":315,"Value":315,"Internal":true,"Count Failed Values":true},{"ID":2411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2412,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":7587798,"Executor Run Time":19,"Executor CPU Time":15817712,"Peak Execution Memory":4456432,"Result Size":6897,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":315,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"164\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"168\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"165\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429609,"Completion Time":1784293429660,"Accumulables":[{"ID":1987,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1990,"Name":"local bytes read","Value":"315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1991,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1992,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2387,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2389,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2390,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2395,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2396,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7587798,"Internal":true,"Count Failed Values":true},{"ID":2397,"Name":"internal.metrics.executorRunTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":2398,"Name":"internal.metrics.executorCpuTime","Value":15817712,"Internal":true,"Count Failed Values":true},{"ID":2399,"Name":"internal.metrics.resultSize","Value":6897,"Internal":true,"Count Failed Values":true},{"ID":2404,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2410,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":315,"Internal":true,"Count Failed Values":true},{"ID":2411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2412,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":19,"Completion Time":1784293429661,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2384,7],[2383,1],[2382,73],[2381,5],[2380,1048616]]}
{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429586,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429724,"Failed":false,"Killed":false,"Accumulables":[{"ID":1761,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"local bytes read","Update":"1031","Value":"1031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2274,"Name":"data size","Update":"240","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2293,"Name":"shuffle bytes written","Update":"420","Value":"420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2294,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2295,"Name":"shuffle write time","Update":"1983478","Value":"1983478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2296,"Name":"duration","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2297,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2298,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2300,"Name":"time in aggregation build","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2303,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2304,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2306,"Name":"time in aggregation build","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2309,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2310,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2312,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2315,"Name":"number of output rows","Update":"60","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2319,"Name":"internal.metrics.executorDeserializeTime","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":2320,"Name":"internal.metrics.executorDeserializeCpuTime","Update":50199940,"Value":50199940,"Internal":true,"Count Failed Values":true},{"ID":2321,"Name":"internal.metrics.executorRunTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":2322,"Name":"internal.metrics.executorCpuTime","Update":68115920,"Value":68115920,"Internal":true,"Count Failed Values":true},{"ID":2323,"Name":"internal.metrics.resultSize","Update":7297,"Value":7297,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.peakExecutionMemory","Update":788384,"Value":788384,"Internal":true,"Count Failed Values":true},{"ID":2330,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2331,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2332,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2333,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2334,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1031,"Value":1031,"Internal":true,"Count Failed Values":true},{"ID":2335,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2336,"Name":"internal.metrics.shuffle.read.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2337,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2338,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2339,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2340,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2341,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2342,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2343,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2344,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2345,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2346,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":420,"Value":420,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.shuffle.write.writeTime","Update":1983478,"Value":1983478,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":57,"Executor Deserialize CPU Time":50199940,"Executor Run Time":75,"Executor CPU Time":68115920,"Peak Execution Memory":788384,"Result Size":7297,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1031,"Total Records Read":18,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":420,"Shuffle Write Time":1983478,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"163\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"154\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429580,"Completion Time":1784293429724,"Accumulables":[{"ID":1761,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1764,"Name":"local bytes read","Value":"1031","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1765,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1766,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2274,"Name":"data size","Value":"240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2293,"Name":"shuffle bytes written","Value":"420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2294,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2295,"Name":"shuffle write time","Value":"1983478","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2296,"Name":"duration","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2297,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2298,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2300,"Name":"time in aggregation build","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2303,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2304,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2306,"Name":"time in aggregation build","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2309,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2310,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2312,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2315,"Name":"number of output rows","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2319,"Name":"internal.metrics.executorDeserializeTime","Value":57,"Internal":true,"Count Failed Values":true},{"ID":2320,"Name":"internal.metrics.executorDeserializeCpuTime","Value":50199940,"Internal":true,"Count Failed Values":true},{"ID":2321,"Name":"internal.metrics.executorRunTime","Value":75,"Internal":true,"Count Failed Values":true},{"ID":2322,"Name":"internal.metrics.executorCpuTime","Value":68115920,"Internal":true,"Count Failed Values":true},{"ID":2323,"Name":"internal.metrics.resultSize","Value":7297,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.peakExecutionMemory","Value":788384,"Internal":true,"Count Failed Values":true},{"ID":2330,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2331,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2332,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2333,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2334,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1031,"Internal":true,"Count Failed Values":true},{"ID":2335,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2336,"Name":"internal.metrics.shuffle.read.recordsRead","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2337,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2338,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2339,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2340,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2341,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2342,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2343,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2344,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2345,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2346,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":420,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.shuffle.write.writeTime","Value":1983478,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":18,"Completion Time":1784293429725,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (70)\n+- == Current Plan ==\n   CollectLimit (44)\n   +- Project (43)\n      +- BroadcastHashJoin Inner BuildRight (42)\n         :- Project (19)\n         :  +- BroadcastHashJoin Inner BuildRight (18)\n         :     :- ShuffleQueryStage (4)\n         :     :  +- Exchange (3)\n         :     :     +- * Project (2)\n         :     :        +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (17), Statistics(sizeInBytes=1024.0 KiB, rowCount=5)\n         :        +- BroadcastExchange (16)\n         :           +- * HashAggregate (15)\n         :              +- AQEShuffleRead (14)\n         :                 +- ShuffleQueryStage (13), Statistics(sizeInBytes=120.0 B, rowCount=5)\n         :                    +- Exchange (12)\n         :                       +- * HashAggregate (11)\n         :                          +- * HashAggregate (10)\n         :                             +- AQEShuffleRead (9)\n         :                                +- ShuffleQueryStage (8), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                                   +- Exchange (7)\n         :                                      +- * HashAggregate (6)\n         :                                         +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- BroadcastQueryStage (41)\n            +- BroadcastExchange (40)\n               +- * HashAggregate (39)\n                  +- AQEShuffleRead (38)\n                     +- ShuffleQueryStage (37), Statistics(sizeInBytes=240.0 B, rowCount=5)\n                        +- Exchange (36)\n                           +- * HashAggregate (35)\n                              +- * HashAggregate (34)\n                                 +- * HashAggregate (33)\n                                    +- * Project (32)\n                                       +- * BroadcastHashJoin LeftOuter BuildRight (31)\n                                          :- AQEShuffleRead (24)\n                                          :  +- ShuffleQueryStage (23), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                          :     +- Exchange (22)\n                                          :        +- * Project (21)\n                                          :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (20)\n                                          +- BroadcastQueryStage (30), Statistics(sizeInBytes=1024.1 KiB, rowCount=57)\n                                             +- BroadcastExchange (29)\n                                                +- AQEShuffleRead (28)\n                                                   +- ShuffleQueryStage (27), Statistics(sizeInBytes=1368.0 B, rowCount=57)\n                                                      +- Exchange (26)\n                                                         +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (25)\n+- == Initial Plan ==\n   CollectLimit (69)\n   +- Project (68)\n      +- SortMergeJoin Inner (67)\n         :- Project (54)\n         :  +- SortMergeJoin Inner (53)\n         :     :- Sort (46)\n         :     :  +- Exchange (45)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (52)\n         :        +- HashAggregate (51)\n         :           +- Exchange (50)\n         :              +- HashAggregate (49)\n         :                 +- HashAggregate (48)\n         :                    +- Exchange (47)\n         :                       +- HashAggregate (6)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (5)\n         +- Sort (66)\n            +- HashAggregate (65)\n               +- Exchange (64)\n                  +- HashAggregate (63)\n                     +- HashAggregate (62)\n                        +- HashAggregate (61)\n                           +- Project (60)\n                              +- SortMergeJoin LeftOuter (59)\n                                 :- Sort (56)\n                                 :  +- Exchange (55)\n                                 :     +- Project (21)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (20)\n                                 +- Sort (58)\n                                    +- Exchange (57)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (25)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(6) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(7) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(12) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(13) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(14) AQEShuffleRead\nInput [2]: [course_id#44, count#399L]\nArguments: coalesced\n\n(15) HashAggregate [codegen id : 7]\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(16) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]\n\n(17) BroadcastQueryStage\nOutput [2]: [course_id#44, unique_users#150L]\nArguments: 7\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(20) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(21) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(22) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(23) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(24) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(25) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(26) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(27) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(28) AQEShuffleRead\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: local\n\n(29) BroadcastExchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]\n\n(30) BroadcastQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 5\n\n(31) BroadcastHashJoin [codegen id : 6]\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(32) Project [codegen id : 6]\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(33) HashAggregate [codegen id : 6]\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(34) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(35) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(36) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]\n\n(37) ShuffleQueryStage\nOutput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: 6\n\n(38) AQEShuffleRead\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: coalesced\n\n(39) HashAggregate [codegen id : 8]\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(40) BroadcastExchange\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1537]\n\n(41) BroadcastQueryStage\nOutput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: 8\n\n(42) BroadcastHashJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(43) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(44) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(45) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(46) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(47) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(48) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(49) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(50) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(51) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(52) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(54) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(55) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(56) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(57) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(58) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(60) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(61) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(62) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(63) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(64) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(65) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(66) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(68) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(69) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(70) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#30], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2392,"metricType":"sum"},{"name":"partition data size","accumulatorId":2393,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2394,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2388,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2389,"metricType":"timing"},{"name":"peak memory","accumulatorId":2387,"metricType":"size"},{"name":"number of output rows","accumulatorId":2386,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2391,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2390,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2385,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2384,"metricType":"timing"},{"name":"time to build","accumulatorId":2383,"metricType":"timing"},{"name":"time to collect","accumulatorId":2382,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2380,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2451,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1537]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#37], [lesson_id#52], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2316,"metricType":"sum"},{"name":"partition data size","accumulatorId":2317,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2318,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2198,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2197,"metricType":"timing"},{"name":"time to build","accumulatorId":2196,"metricType":"timing"},{"name":"time to collect","accumulatorId":2195,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2194,"metricType":"sum"},{"name":"data size","accumulatorId":2193,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2315,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2311,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2312,"metricType":"timing"},{"name":"peak memory","accumulatorId":2310,"metricType":"size"},{"name":"number of output rows","accumulatorId":2309,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2314,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2313,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2305,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2306,"metricType":"timing"},{"name":"peak memory","accumulatorId":2304,"metricType":"size"},{"name":"number of output rows","accumulatorId":2303,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2308,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2307,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2299,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2300,"metricType":"timing"},{"name":"peak memory","accumulatorId":2298,"metricType":"size"},{"name":"number of output rows","accumulatorId":2297,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2302,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2301,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2296,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2294,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2295,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2289,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2286,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2283,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2292,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2285,"metricType":"sum"},{"name":"records read","accumulatorId":2282,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2280,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2281,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2278,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2284,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2277,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2276,"metricType":"sum"},{"name":"data size","accumulatorId":2274,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2290,"metricType":"size"},{"name":"number of partitions","accumulatorId":2275,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2291,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2279,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2293,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2464,"metricType":"sum"},{"name":"partition data size","accumulatorId":2465,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2466,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2460,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2461,"metricType":"timing"},{"name":"peak memory","accumulatorId":2459,"metricType":"size"},{"name":"number of output rows","accumulatorId":2458,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2463,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2462,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2457,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2456,"metricType":"timing"},{"name":"time to build","accumulatorId":2455,"metricType":"timing"},{"name":"time to collect","accumulatorId":2454,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2453,"metricType":"sum"},{"name":"data size","accumulatorId":2452,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2450,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2448,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2442,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2449,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2443,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2440,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2437,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2446,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2439,"metricType":"sum"},{"name":"records read","accumulatorId":2436,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2434,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2435,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2432,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2438,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2431,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2441,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2430,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2444,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2445,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2433,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2447,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2464,1],[2466,1],[2465,440]]}
{"Event":"SparkListenerJobStart","Job ID":20,"Submission Time":1784293429756,"Stage Infos":[{"Stage ID":39,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"134\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":59,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"153\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[58],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"163\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"154\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"178\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"182\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[40],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[39,40,41],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"178\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId d3116dc2-c8cb-4215-ac07-fb730a04812f)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"178\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"182\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[40],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429756,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"178\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId d3116dc2-c8cb-4215-ac07-fb730a04812f)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":41,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429764,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":41,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429764,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293429796,"Failed":false,"Killed":false,"Accumulables":[{"ID":2277,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2280,"Name":"local bytes read","Update":"420","Value":"420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2281,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2282,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2457,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2458,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2459,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2461,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2462,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2467,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6970503,"Value":6970503,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.executorRunTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.executorCpuTime","Update":13026389,"Value":13026389,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.resultSize","Update":8721,"Value":8721,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2480,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2481,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2482,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":420,"Value":420,"Internal":true,"Count Failed Values":true},{"ID":2483,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2484,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2485,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2486,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2487,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":6970503,"Executor Run Time":14,"Executor CPU Time":13026389,"Peak Execution Memory":4456432,"Result Size":8721,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":420,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"178\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"179\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[63],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":63,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"182\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[40],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429756,"Completion Time":1784293429797,"Accumulables":[{"ID":2277,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2280,"Name":"local bytes read","Value":"420","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2281,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2282,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2457,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2458,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2459,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2461,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2462,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2467,"Name":"internal.metrics.executorDeserializeTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6970503,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.executorRunTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.executorCpuTime","Value":13026389,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.resultSize","Value":8721,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2480,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2481,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2482,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":420,"Internal":true,"Count Failed Values":true},{"ID":2483,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2484,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2485,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2486,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2487,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2493,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2494,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":20,"Completion Time":1784293429797,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2456,3],[2455,1],[2454,62],[2453,5],[2452,1048616]]}
{"Event":"SparkListenerTaskEnd","Stage ID":26,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293429284,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431571,"Failed":false,"Killed":false,"Accumulables":[{"ID":1412,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1615,"Name":"data size","Update":"304","Value":"304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1634,"Name":"shuffle bytes written","Update":"499","Value":"499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1635,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1636,"Name":"shuffle write time","Update":"10126516","Value":"10126516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1637,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1807,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":1808,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6273036,"Value":6273036,"Internal":true,"Count Failed Values":true},{"ID":1809,"Name":"internal.metrics.executorRunTime","Update":2266,"Value":2266,"Internal":true,"Count Failed Values":true},{"ID":1810,"Name":"internal.metrics.executorCpuTime","Update":25629041,"Value":25629041,"Internal":true,"Count Failed Values":true},{"ID":1811,"Name":"internal.metrics.resultSize","Update":1983,"Value":1983,"Internal":true,"Count Failed Values":true},{"ID":1812,"Name":"internal.metrics.jvmGCTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1835,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":499,"Value":499,"Internal":true,"Count Failed Values":true},{"ID":1836,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":1837,"Name":"internal.metrics.shuffle.write.writeTime","Update":10126516,"Value":10126516,"Internal":true,"Count Failed Values":true},{"ID":1839,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":6273036,"Executor Run Time":2266,"Executor CPU Time":25629041,"Peak Execution Memory":0,"Result Size":1983,"JVM GC Time":15,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":499,"Shuffle Write Time":10126516,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293429281,"Completion Time":1784293431573,"Accumulables":[{"ID":1412,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1615,"Name":"data size","Value":"304","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1634,"Name":"shuffle bytes written","Value":"499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1635,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1636,"Name":"shuffle write time","Value":"10126516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1637,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1807,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":1808,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6273036,"Internal":true,"Count Failed Values":true},{"ID":1809,"Name":"internal.metrics.executorRunTime","Value":2266,"Internal":true,"Count Failed Values":true},{"ID":1810,"Name":"internal.metrics.executorCpuTime","Value":25629041,"Internal":true,"Count Failed Values":true},{"ID":1811,"Name":"internal.metrics.resultSize","Value":1983,"Internal":true,"Count Failed Values":true},{"ID":1812,"Name":"internal.metrics.jvmGCTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1835,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":499,"Internal":true,"Count Failed Values":true},{"ID":1836,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":1837,"Name":"internal.metrics.shuffle.write.writeTime","Value":10126516,"Internal":true,"Count Failed Values":true},{"ID":1839,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":12,"Completion Time":1784293431574,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (71)\n+- == Final Plan ==\n   CollectLimit (45)\n   +- * Project (44)\n      +- * BroadcastHashJoin Inner BuildRight (43)\n         :- * Project (20)\n         :  +- * BroadcastHashJoin Inner BuildRight (19)\n         :     :- AQEShuffleRead (5)\n         :     :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=304.0 B, rowCount=5)\n         :     :     +- Exchange (3)\n         :     :        +- * Project (2)\n         :     :           +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (18), Statistics(sizeInBytes=1024.0 KiB, rowCount=5)\n         :        +- BroadcastExchange (17)\n         :           +- * HashAggregate (16)\n         :              +- AQEShuffleRead (15)\n         :                 +- ShuffleQueryStage (14), Statistics(sizeInBytes=120.0 B, rowCount=5)\n         :                    +- Exchange (13)\n         :                       +- * HashAggregate (12)\n         :                          +- * HashAggregate (11)\n         :                             +- AQEShuffleRead (10)\n         :                                +- ShuffleQueryStage (9), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n         :                                   +- Exchange (8)\n         :                                      +- * HashAggregate (7)\n         :                                         +- * Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         +- BroadcastQueryStage (42), Statistics(sizeInBytes=1024.0 KiB, rowCount=5)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=240.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * HashAggregate (35)\n                                 +- * HashAggregate (34)\n                                    +- * Project (33)\n                                       +- * BroadcastHashJoin LeftOuter BuildRight (32)\n                                          :- AQEShuffleRead (25)\n                                          :  +- ShuffleQueryStage (24), Statistics(sizeInBytes=432.0 B, rowCount=18)\n                                          :     +- Exchange (23)\n                                          :        +- * Project (22)\n                                          :           +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (21)\n                                          +- BroadcastQueryStage (31), Statistics(sizeInBytes=1024.1 KiB, rowCount=57)\n                                             +- BroadcastExchange (30)\n                                                +- AQEShuffleRead (29)\n                                                   +- ShuffleQueryStage (28), Statistics(sizeInBytes=1368.0 B, rowCount=57)\n                                                      +- Exchange (27)\n                                                         +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (26)\n+- == Initial Plan ==\n   CollectLimit (70)\n   +- Project (69)\n      +- SortMergeJoin Inner (68)\n         :- Project (55)\n         :  +- SortMergeJoin Inner (54)\n         :     :- Sort (47)\n         :     :  +- Exchange (46)\n         :     :     +- Project (2)\n         :     :        +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (1)\n         :     +- Sort (53)\n         :        +- HashAggregate (52)\n         :           +- Exchange (51)\n         :              +- HashAggregate (50)\n         :                 +- HashAggregate (49)\n         :                    +- Exchange (48)\n         :                       +- HashAggregate (7)\n         :                          +- Scan JDBCRelation(raw.enrollments) [numPartitions=1]  (6)\n         +- Sort (67)\n            +- HashAggregate (66)\n               +- Exchange (65)\n                  +- HashAggregate (64)\n                     +- HashAggregate (63)\n                        +- HashAggregate (62)\n                           +- Project (61)\n                              +- SortMergeJoin LeftOuter (60)\n                                 :- Sort (57)\n                                 :  +- Exchange (56)\n                                 :     +- Project (22)\n                                 :        +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (21)\n                                 +- Sort (59)\n                                    +- Exchange (58)\n                                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (26)\n\n\n(1) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [id#16, title#17]\nPushedFilters: [*IsNotNull(id)]\nReadSchema: struct<id:int,title:string>\n\n(2) Project [codegen id : 1]\nOutput [2]: [id#16 AS course_id#24, title#17]\nInput [2]: [id#16, title#17]\n\n(3) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]\n\n(4) ShuffleQueryStage\nOutput [2]: [course_id#24, title#17]\nArguments: 0\n\n(5) AQEShuffleRead\nInput [2]: [course_id#24, title#17]\nArguments: local\n\n(6) Scan JDBCRelation(raw.enrollments) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#43, course_id#44]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<user_id:int,course_id:int>\n\n(7) HashAggregate [codegen id : 2]\nInput [2]: [user_id#43, course_id#44]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(8) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]\n\n(9) ShuffleQueryStage\nOutput [2]: [course_id#44, user_id#43]\nArguments: 1\n\n(10) AQEShuffleRead\nInput [2]: [course_id#44, user_id#43]\nArguments: coalesced\n\n(11) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(12) HashAggregate [codegen id : 5]\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(13) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]\n\n(14) ShuffleQueryStage\nOutput [2]: [course_id#44, count#399L]\nArguments: 4\n\n(15) AQEShuffleRead\nInput [2]: [course_id#44, count#399L]\nArguments: coalesced\n\n(16) HashAggregate [codegen id : 7]\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(17) BroadcastExchange\nInput [2]: [course_id#44, unique_users#150L]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]\n\n(18) BroadcastQueryStage\nOutput [2]: [course_id#44, unique_users#150L]\nArguments: 7\n\n(19) BroadcastHashJoin [codegen id : 9]\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(20) Project [codegen id : 9]\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(21) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [id#29, course_id#30]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<id:int,course_id:int>\n\n(22) Project [codegen id : 3]\nOutput [2]: [id#29 AS lesson_id#37, course_id#30]\nInput [2]: [id#29, course_id#30]\n\n(23) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]\n\n(24) ShuffleQueryStage\nOutput [2]: [lesson_id#37, course_id#30]\nArguments: 2\n\n(25) AQEShuffleRead\nInput [2]: [lesson_id#37, course_id#30]\nArguments: coalesced\n\n(26) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [2]: [lesson_id#52, viewed_at#53]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<lesson_id:int,viewed_at:timestamp>\n\n(27) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]\n\n(28) ShuffleQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 3\n\n(29) AQEShuffleRead\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: local\n\n(30) BroadcastExchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]\n\n(31) BroadcastQueryStage\nOutput [2]: [lesson_id#52, viewed_at#53]\nArguments: 5\n\n(32) BroadcastHashJoin [codegen id : 6]\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project [codegen id : 6]\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(34) HashAggregate [codegen id : 6]\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(35) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(36) HashAggregate [codegen id : 6]\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(37) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]\n\n(38) ShuffleQueryStage\nOutput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: 6\n\n(39) AQEShuffleRead\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 8]\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(41) BroadcastExchange\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1537]\n\n(42) BroadcastQueryStage\nOutput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: 8\n\n(43) BroadcastHashJoin [codegen id : 9]\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(44) Project [codegen id : 9]\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(45) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(46) Exchange\nInput [2]: [course_id#24, title#17]\nArguments: hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1019]\n\n(47) Sort\nInput [2]: [course_id#24, title#17]\nArguments: [course_id#24 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [course_id#44, user_id#43]\nArguments: hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1011]\n\n(49) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [2]: [course_id#44, user_id#43]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#44, user_id#43]\n\n(50) HashAggregate\nInput [2]: [course_id#44, user_id#43]\nKeys [1]: [course_id#44]\nFunctions [1]: [partial_count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count#399L]\n\n(51) Exchange\nInput [2]: [course_id#44, count#399L]\nArguments: hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1015]\n\n(52) HashAggregate\nInput [2]: [course_id#44, count#399L]\nKeys [1]: [course_id#44]\nFunctions [1]: [count(distinct user_id#43)]\nAggregate Attributes [1]: [count(user_id#43)#151L]\nResults [2]: [course_id#44, count(user_id#43)#151L AS unique_users#150L]\n\n(53) Sort\nInput [2]: [course_id#44, unique_users#150L]\nArguments: [course_id#44 ASC NULLS FIRST], false, 0\n\n(54) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#44]\nJoin type: Inner\nJoin condition: None\n\n(55) Project\nOutput [3]: [course_id#24, title#17, unique_users#150L]\nInput [4]: [course_id#24, title#17, course_id#44, unique_users#150L]\n\n(56) Exchange\nInput [2]: [lesson_id#37, course_id#30]\nArguments: hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1025]\n\n(57) Sort\nInput [2]: [lesson_id#37, course_id#30]\nArguments: [lesson_id#37 ASC NULLS FIRST], false, 0\n\n(58) Exchange\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1026]\n\n(59) Sort\nInput [2]: [lesson_id#52, viewed_at#53]\nArguments: [lesson_id#52 ASC NULLS FIRST], false, 0\n\n(60) SortMergeJoin\nLeft keys [1]: [lesson_id#37]\nRight keys [1]: [lesson_id#52]\nJoin type: LeftOuter\nJoin condition: None\n\n(61) Project\nOutput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nInput [4]: [lesson_id#37, course_id#30, lesson_id#52, viewed_at#53]\n\n(62) HashAggregate\nInput [4]: [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(63) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [2]: [course_id#30, lesson_id#37]\nFunctions [3]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)]\nAggregate Attributes [3]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173]\nResults [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\n\n(64) HashAggregate\nInput [5]: [course_id#30, lesson_id#37, count#402L, min#404, max#406]\nKeys [1]: [course_id#30]\nFunctions [4]: [merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\n\n(65) Exchange\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nArguments: hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1035]\n\n(66) HashAggregate\nInput [5]: [course_id#30, count#402L, min#404, max#406, count#409L]\nKeys [1]: [course_id#30]\nFunctions [4]: [count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)]\nAggregate Attributes [4]: [count(lesson_id#52)#169L, min(viewed_at#53)#171, max(viewed_at#53)#173, count(lesson_id#37)#175L]\nResults [5]: [course_id#30, count(lesson_id#37)#175L AS lessons_count#168L, count(lesson_id#52)#169L AS total_views#170L, min(viewed_at#53)#171 AS first_view#172, max(viewed_at#53)#173 AS last_view#174]\n\n(67) Sort\nInput [5]: [course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\nArguments: [course_id#30 ASC NULLS FIRST], false, 0\n\n(68) SortMergeJoin\nLeft keys [1]: [course_id#24]\nRight keys [1]: [course_id#30]\nJoin type: Inner\nJoin condition: None\n\n(69) Project\nOutput [8]: [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]\nInput [8]: [course_id#24, title#17, unique_users#150L, course_id#30, lessons_count#168L, total_views#170L, first_view#172, last_view#174]\n\n(70) CollectLimit\nInput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: 21\n\n(71) AdaptiveSparkPlan\nOutput [8]: [toprettystring(course_id)#381, toprettystring(title)#382, toprettystring(unique_users)#383, toprettystring(lessons_count)#384, toprettystring(total_views)#385, toprettystring(avg_views_per_user)#386, toprettystring(first_view)#387, toprettystring(last_view)#388]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(course_id#24, Some(Europe/Samara)) AS toprettystring(course_id)#381, toprettystring(title#17, Some(Europe/Samara)) AS toprettystring(title)#382, toprettystring(unique_users#150L, Some(Europe/Samara)) AS toprettystring(unique_users)#383, toprettystring(lessons_count#168L, Some(Europe/Samara)) AS toprettystring(lessons_count)#384, toprettystring(total_views#170L, Some(Europe/Samara)) AS toprettystring(total_views)#385, toprettystring(round((cast(total_views#170L as double) / cast(unique_users#150L as double)), 2), Some(Europe/Samara)) AS toprettystring(avg_views_per_user)#386, toprettystring(first_view#172, Some(Europe/Samara)) AS toprettystring(first_view)#387, toprettystring(last_view#174, Some(Europe/Samara)) AS toprettystring(last_view)#388]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#30], Inner, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#24, title#17, unique_users#150L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#24], [course_id#44], Inner, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#24, 200), ENSURE_REQUIREMENTS, [plan_id=1049]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Project","simpleString":"Project [id#16 AS course_id#24, title#17]","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [id#16,title#17] PushedFilters: [*IsNotNull(id)], ReadSchema: struct<id:int,title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1412,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1637,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1635,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1629,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1636,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1630,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1627,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1624,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1633,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1626,"metricType":"sum"},{"name":"records read","accumulatorId":1623,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1621,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1622,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1619,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1625,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1618,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1628,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1617,"metricType":"sum"},{"name":"data size","accumulatorId":1615,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1631,"metricType":"size"},{"name":"number of partitions","accumulatorId":1616,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1632,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1620,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2525,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1466]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[count(distinct user_id#43)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, 200), ENSURE_REQUIREMENTS, [plan_id=1202]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44], functions=[partial_count(distinct user_id#43)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#44, user_id#43, 200), ENSURE_REQUIREMENTS, [plan_id=1067]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#44, user_id#43], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.enrollments) [numPartitions=1] [user_id#43,course_id#44] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<user_id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1484,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1480,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1481,"metricType":"timing"},{"name":"peak memory","accumulatorId":1479,"metricType":"size"},{"name":"number of output rows","accumulatorId":1478,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1483,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1482,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1703,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1701,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1695,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1702,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1696,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1693,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1690,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1699,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1692,"metricType":"sum"},{"name":"records read","accumulatorId":1689,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1687,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1688,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1685,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1691,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1684,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1694,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1683,"metricType":"sum"},{"name":"data size","accumulatorId":1681,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1697,"metricType":"size"},{"name":"number of partitions","accumulatorId":1682,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1698,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1686,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1700,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2019,"metricType":"sum"},{"name":"partition data size","accumulatorId":2020,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2021,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2015,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2016,"metricType":"timing"},{"name":"peak memory","accumulatorId":2014,"metricType":"size"},{"name":"number of output rows","accumulatorId":2013,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2018,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2017,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2009,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2010,"metricType":"timing"},{"name":"peak memory","accumulatorId":2008,"metricType":"size"},{"name":"number of output rows","accumulatorId":2007,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2012,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2011,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2006,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2004,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1998,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2005,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1999,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1996,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1993,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2002,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1995,"metricType":"sum"},{"name":"records read","accumulatorId":1992,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1990,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1991,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1988,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1994,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1987,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1997,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1986,"metricType":"sum"},{"name":"data size","accumulatorId":1984,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2000,"metricType":"size"},{"name":"number of partitions","accumulatorId":1985,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2001,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1989,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2003,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2392,"metricType":"sum"},{"name":"partition data size","accumulatorId":2393,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2394,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2388,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2389,"metricType":"timing"},{"name":"peak memory","accumulatorId":2387,"metricType":"size"},{"name":"number of output rows","accumulatorId":2386,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2391,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2390,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2385,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2384,"metricType":"timing"},{"name":"time to build","accumulatorId":2383,"metricType":"timing"},{"name":"time to collect","accumulatorId":2382,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2380,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2524,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1537]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[count(lesson_id#52), min(viewed_at#53), max(viewed_at#53), count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#30, 200), ENSURE_REQUIREMENTS, [plan_id=1408]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53), partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[merge_count(lesson_id#52), merge_min(viewed_at#53), merge_max(viewed_at#53)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#30, lesson_id#37], functions=[partial_count(lesson_id#52), partial_min(viewed_at#53), partial_max(viewed_at#53)])","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#37, course_id#30, viewed_at#53, lesson_id#52]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#37], [lesson_id#52], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1104]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Project","simpleString":"Project [id#29 AS lesson_id#37, course_id#30]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#29,course_id#30] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1780,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1778,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1772,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1779,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1773,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1770,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1767,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1776,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1769,"metricType":"sum"},{"name":"records read","accumulatorId":1766,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1764,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1765,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1762,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1768,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1761,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1771,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1760,"metricType":"sum"},{"name":"data size","accumulatorId":1758,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1774,"metricType":"size"},{"name":"number of partitions","accumulatorId":1759,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1775,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1763,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1777,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2316,"metricType":"sum"},{"name":"partition data size","accumulatorId":2317,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2318,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1311]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#52, 200), ENSURE_REQUIREMENTS, [plan_id=1111]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#52,viewed_at#53] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<lesson_id:int,viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1587,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1806,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1804,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1798,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1805,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1799,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1796,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1793,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1802,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1795,"metricType":"sum"},{"name":"records read","accumulatorId":1792,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1790,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1791,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1788,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1794,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1787,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1797,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1786,"metricType":"sum"},{"name":"data size","accumulatorId":1784,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1800,"metricType":"size"},{"name":"number of partitions","accumulatorId":1785,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1801,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1789,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1803,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2198,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2197,"metricType":"timing"},{"name":"time to build","accumulatorId":2196,"metricType":"timing"},{"name":"time to collect","accumulatorId":2195,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2194,"metricType":"sum"},{"name":"data size","accumulatorId":2193,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2315,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2311,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2312,"metricType":"timing"},{"name":"peak memory","accumulatorId":2310,"metricType":"size"},{"name":"number of output rows","accumulatorId":2309,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2314,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2313,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2305,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2306,"metricType":"timing"},{"name":"peak memory","accumulatorId":2304,"metricType":"size"},{"name":"number of output rows","accumulatorId":2303,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2308,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2307,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2299,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2300,"metricType":"timing"},{"name":"peak memory","accumulatorId":2298,"metricType":"size"},{"name":"number of output rows","accumulatorId":2297,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2302,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2301,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2296,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2294,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2295,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2289,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2286,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2283,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2292,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2285,"metricType":"sum"},{"name":"records read","accumulatorId":2282,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2280,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2281,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2278,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2284,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2277,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2276,"metricType":"sum"},{"name":"data size","accumulatorId":2274,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2290,"metricType":"size"},{"name":"number of partitions","accumulatorId":2275,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2291,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2279,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2293,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2464,"metricType":"sum"},{"name":"partition data size","accumulatorId":2465,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2466,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2460,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2461,"metricType":"timing"},{"name":"peak memory","accumulatorId":2459,"metricType":"size"},{"name":"number of output rows","accumulatorId":2458,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2463,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2462,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2457,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2456,"metricType":"timing"},{"name":"time to build","accumulatorId":2455,"metricType":"timing"},{"name":"time to collect","accumulatorId":2454,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2453,"metricType":"sum"},{"name":"data size","accumulatorId":2452,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2523,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2511,"metricType":"sum"},{"name":"records read","accumulatorId":2508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2502,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2516,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2519,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[2525,1]]}
{"Event":"SparkListenerJobStart","Job ID":21,"Submission Time":1784293431614,"Stage Infos":[{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":45,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"192\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[42],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[42,43],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"192\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[42],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431614,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":43,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431618,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":43,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431618,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431650,"Failed":false,"Killed":false,"Accumulables":[{"ID":1618,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1621,"Name":"local bytes read","Update":"499","Value":"499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1622,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1623,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2522,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2523,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2524,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2526,"Name":"internal.metrics.executorDeserializeTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.executorDeserializeCpuTime","Update":3973830,"Value":3973830,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.executorRunTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.executorCpuTime","Update":15871740,"Value":15871740,"Internal":true,"Count Failed Values":true},{"ID":2530,"Name":"internal.metrics.resultSize","Update":4554,"Value":4554,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.peakExecutionMemory","Update":520,"Value":520,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2540,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2541,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":499,"Value":499,"Internal":true,"Count Failed Values":true},{"ID":2542,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2543,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2544,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2545,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2546,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2547,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2548,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2549,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2550,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2551,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2552,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2553,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":6,"Executor Deserialize CPU Time":3973830,"Executor Run Time":21,"Executor CPU Time":15871740,"Peak Execution Memory":520,"Result Size":4554,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":499,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"207\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"192\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[66],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":66,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"206\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[42],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431614,"Completion Time":1784293431650,"Accumulables":[{"ID":1618,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1621,"Name":"local bytes read","Value":"499","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1622,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1623,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2522,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2523,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2524,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2526,"Name":"internal.metrics.executorDeserializeTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":2527,"Name":"internal.metrics.executorDeserializeCpuTime","Value":3973830,"Internal":true,"Count Failed Values":true},{"ID":2528,"Name":"internal.metrics.executorRunTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":2529,"Name":"internal.metrics.executorCpuTime","Value":15871740,"Internal":true,"Count Failed Values":true},{"ID":2530,"Name":"internal.metrics.resultSize","Value":4554,"Internal":true,"Count Failed Values":true},{"ID":2535,"Name":"internal.metrics.peakExecutionMemory","Value":520,"Internal":true,"Count Failed Values":true},{"ID":2537,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2538,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2539,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2540,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2541,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":499,"Internal":true,"Count Failed Values":true},{"ID":2542,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2543,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2544,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2545,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2546,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2547,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2548,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2549,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2550,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2551,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2552,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2553,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":21,"Completion Time":1784293431650,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1784293431656,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"rootExecutionId":2,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (49)\n+- Project (48)\n   +- BroadcastNestedLoopJoin Cross BuildRight (47)\n      :- BroadcastNestedLoopJoin Cross BuildRight (38)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (25)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (16)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (10)\n      :  :  :  :  :- HashAggregate (4)\n      :  :  :  :  :  +- Exchange (3)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (9)\n      :  :  :  :     +- HashAggregate (8)\n      :  :  :  :        +- Exchange (7)\n      :  :  :  :           +- HashAggregate (6)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (5)\n      :  :  :  +- BroadcastExchange (15)\n      :  :  :     +- HashAggregate (14)\n      :  :  :        +- Exchange (13)\n      :  :  :           +- HashAggregate (12)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (11)\n      :  :  +- BroadcastExchange (24)\n      :  :     +- HashAggregate (23)\n      :  :        +- Exchange (22)\n      :  :           +- HashAggregate (21)\n      :  :              +- HashAggregate (20)\n      :  :                 +- Exchange (19)\n      :  :                    +- HashAggregate (18)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (17)\n      :  +- BroadcastExchange (37)\n      :     +- HashAggregate (36)\n      :        +- Exchange (35)\n      :           +- HashAggregate (34)\n      :              +- HashAggregate (33)\n      :                 +- Exchange (32)\n      :                    +- HashAggregate (31)\n      :                       +- HashAggregate (30)\n      :                          +- Exchange (29)\n      :                             +- HashAggregate (28)\n      :                                +- Project (27)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (26)\n      +- BroadcastExchange (46)\n         +- HashAggregate (45)\n            +- Exchange (44)\n               +- HashAggregate (43)\n                  +- HashAggregate (42)\n                     +- Exchange (41)\n                        +- HashAggregate (40)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (39)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1] \nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(4) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(5) Scan JDBCRelation(raw.courses) [numPartitions=1] \nOutput: []\nReadSchema: struct<>\n\n(6) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(7) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(8) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(9) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(10) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(11) Scan JDBCRelation(raw.lessons) [numPartitions=1] \nOutput: []\nReadSchema: struct<>\n\n(12) HashAggregate\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(13) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(14) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(15) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(16) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(17) Scan JDBCRelation(raw.lesson_views) [numPartitions=1] \nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(18) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(19) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(20) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(21) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(22) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(23) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(24) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(25) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(26) Scan JDBCRelation(raw.lessons) [numPartitions=1] \nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(27) Project\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(28) HashAggregate\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(29) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(30) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(31) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(32) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(33) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(34) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(35) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(36) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(37) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(38) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(39) Scan JDBCRelation(raw.lesson_views) [numPartitions=1] \nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(40) HashAggregate\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(41) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(42) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(43) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(44) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(45) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(46) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(47) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(48) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(49) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2592,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2586,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2593,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2587,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2584,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2581,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2590,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2583,"metricType":"sum"},{"name":"records read","accumulatorId":2580,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2578,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2579,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2576,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2582,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2575,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2585,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2574,"metricType":"sum"},{"name":"data size","accumulatorId":2572,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2588,"metricType":"size"},{"name":"number of partitions","accumulatorId":2573,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2589,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2577,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2591,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2569,"metricType":"timing"},{"name":"peak memory","accumulatorId":2567,"metricType":"size"},{"name":"number of output rows","accumulatorId":2566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2570,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1730]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2632,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2626,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2633,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2627,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2624,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2621,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2630,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2623,"metricType":"sum"},{"name":"records read","accumulatorId":2620,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2618,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2619,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2616,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2622,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2615,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2625,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2614,"metricType":"sum"},{"name":"data size","accumulatorId":2612,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2628,"metricType":"size"},{"name":"number of partitions","accumulatorId":2613,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2629,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2617,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2631,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2608,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2609,"metricType":"timing"},{"name":"peak memory","accumulatorId":2607,"metricType":"size"},{"name":"number of output rows","accumulatorId":2606,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2611,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2610,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2605,"metricType":"timing"},{"name":"time to build","accumulatorId":2604,"metricType":"timing"},{"name":"time to collect","accumulatorId":2603,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2602,"metricType":"sum"},{"name":"data size","accumulatorId":2601,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2565,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1735]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2672,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2666,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2673,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2667,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2664,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2661,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2670,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2663,"metricType":"sum"},{"name":"records read","accumulatorId":2660,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2658,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2659,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2656,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2662,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2655,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2665,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2654,"metricType":"sum"},{"name":"data size","accumulatorId":2652,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2668,"metricType":"size"},{"name":"number of partitions","accumulatorId":2653,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2669,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2657,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2671,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2648,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2649,"metricType":"timing"},{"name":"peak memory","accumulatorId":2647,"metricType":"size"},{"name":"number of output rows","accumulatorId":2646,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2651,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2650,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2645,"metricType":"timing"},{"name":"time to build","accumulatorId":2644,"metricType":"timing"},{"name":"time to collect","accumulatorId":2643,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2642,"metricType":"sum"},{"name":"data size","accumulatorId":2641,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2564,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1744]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2746,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2740,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2747,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2741,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2738,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2735,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2744,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2737,"metricType":"sum"},{"name":"records read","accumulatorId":2734,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2732,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2733,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2730,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2736,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2729,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2739,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2728,"metricType":"sum"},{"name":"data size","accumulatorId":2726,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2742,"metricType":"size"},{"name":"number of partitions","accumulatorId":2727,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2743,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2731,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2745,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2722,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2723,"metricType":"timing"},{"name":"peak memory","accumulatorId":2721,"metricType":"size"},{"name":"number of output rows","accumulatorId":2720,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2725,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2724,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2717,"metricType":"timing"},{"name":"peak memory","accumulatorId":2715,"metricType":"size"},{"name":"number of output rows","accumulatorId":2714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2712,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2706,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2713,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2707,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2704,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2701,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2710,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2703,"metricType":"sum"},{"name":"records read","accumulatorId":2700,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2698,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2699,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2696,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2702,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2695,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2705,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2694,"metricType":"sum"},{"name":"data size","accumulatorId":2692,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2708,"metricType":"size"},{"name":"number of partitions","accumulatorId":2693,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2709,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2697,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2711,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2688,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2689,"metricType":"timing"},{"name":"peak memory","accumulatorId":2687,"metricType":"size"},{"name":"number of output rows","accumulatorId":2686,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2691,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2690,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2685,"metricType":"timing"},{"name":"time to build","accumulatorId":2684,"metricType":"timing"},{"name":"time to collect","accumulatorId":2683,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2682,"metricType":"sum"},{"name":"data size","accumulatorId":2681,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2563,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1757]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2854,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2848,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2855,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2849,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2846,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2843,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2852,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2845,"metricType":"sum"},{"name":"records read","accumulatorId":2842,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2840,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2841,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2838,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2844,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2837,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2847,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2836,"metricType":"sum"},{"name":"data size","accumulatorId":2834,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2850,"metricType":"size"},{"name":"number of partitions","accumulatorId":2835,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2851,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2839,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2853,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2831,"metricType":"timing"},{"name":"peak memory","accumulatorId":2829,"metricType":"size"},{"name":"number of output rows","accumulatorId":2828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2825,"metricType":"timing"},{"name":"peak memory","accumulatorId":2823,"metricType":"size"},{"name":"number of output rows","accumulatorId":2822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2820,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2814,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2821,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2815,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2812,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2809,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2818,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2811,"metricType":"sum"},{"name":"records read","accumulatorId":2808,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2806,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2807,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2804,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2810,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2803,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2813,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2802,"metricType":"sum"},{"name":"data size","accumulatorId":2800,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2816,"metricType":"size"},{"name":"number of partitions","accumulatorId":2801,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2817,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2805,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2819,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2796,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2797,"metricType":"timing"},{"name":"peak memory","accumulatorId":2795,"metricType":"size"},{"name":"number of output rows","accumulatorId":2794,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2799,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2798,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2790,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2791,"metricType":"timing"},{"name":"peak memory","accumulatorId":2789,"metricType":"size"},{"name":"number of output rows","accumulatorId":2788,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2793,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2792,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2786,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2780,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2787,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2781,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2778,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2775,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2784,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2777,"metricType":"sum"},{"name":"records read","accumulatorId":2774,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2772,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2773,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2770,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2776,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2769,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2779,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2768,"metricType":"sum"},{"name":"data size","accumulatorId":2766,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2782,"metricType":"size"},{"name":"number of partitions","accumulatorId":2767,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2783,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2771,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2785,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2762,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2763,"metricType":"timing"},{"name":"peak memory","accumulatorId":2761,"metricType":"size"},{"name":"number of output rows","accumulatorId":2760,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2765,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2764,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2759,"metricType":"timing"},{"name":"time to build","accumulatorId":2758,"metricType":"timing"},{"name":"time to collect","accumulatorId":2757,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2756,"metricType":"sum"},{"name":"data size","accumulatorId":2755,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2562,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1766]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2928,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2922,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2929,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2923,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2920,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2917,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2926,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2919,"metricType":"sum"},{"name":"records read","accumulatorId":2916,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2914,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2915,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2912,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2918,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2911,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2921,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2910,"metricType":"sum"},{"name":"data size","accumulatorId":2908,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2924,"metricType":"size"},{"name":"number of partitions","accumulatorId":2909,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2925,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2913,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2927,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2904,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2905,"metricType":"timing"},{"name":"peak memory","accumulatorId":2903,"metricType":"size"},{"name":"number of output rows","accumulatorId":2902,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2907,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2906,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2898,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2899,"metricType":"timing"},{"name":"peak memory","accumulatorId":2897,"metricType":"size"},{"name":"number of output rows","accumulatorId":2896,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2901,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2900,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2894,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2888,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2895,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2889,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2886,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2883,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2892,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2885,"metricType":"sum"},{"name":"records read","accumulatorId":2882,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2880,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2881,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2878,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2884,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2877,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2887,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2876,"metricType":"sum"},{"name":"data size","accumulatorId":2874,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2890,"metricType":"size"},{"name":"number of partitions","accumulatorId":2875,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2891,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2879,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2893,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2870,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2871,"metricType":"timing"},{"name":"peak memory","accumulatorId":2869,"metricType":"size"},{"name":"number of output rows","accumulatorId":2868,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2873,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2872,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2867,"metricType":"timing"},{"name":"time to build","accumulatorId":2866,"metricType":"timing"},{"name":"time to collect","accumulatorId":2865,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2864,"metricType":"sum"},{"name":"data size","accumulatorId":2863,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2561,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1784293431687,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (90)\n+- == Current Plan ==\n   Project (54)\n   +- BroadcastNestedLoopJoin Cross BuildRight (53)\n      :- BroadcastNestedLoopJoin Cross BuildRight (43)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (29)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (19)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (18)\n      :  :  :     +- HashAggregate (17)\n      :  :  :        +- ShuffleQueryStage (16)\n      :  :  :           +- Exchange (15)\n      :  :  :              +- * HashAggregate (14)\n      :  :  :                 +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (28)\n      :  :     +- HashAggregate (27)\n      :  :        +- Exchange (26)\n      :  :           +- HashAggregate (25)\n      :  :              +- HashAggregate (24)\n      :  :                 +- ShuffleQueryStage (23)\n      :  :                    +- Exchange (22)\n      :  :                       +- * HashAggregate (21)\n      :  :                          +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (20)\n      :  +- BroadcastExchange (42)\n      :     +- HashAggregate (41)\n      :        +- Exchange (40)\n      :           +- HashAggregate (39)\n      :              +- HashAggregate (38)\n      :                 +- Exchange (37)\n      :                    +- HashAggregate (36)\n      :                       +- HashAggregate (35)\n      :                          +- ShuffleQueryStage (34)\n      :                             +- Exchange (33)\n      :                                +- * HashAggregate (32)\n      :                                   +- * Project (31)\n      :                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (30)\n      +- BroadcastExchange (52)\n         +- HashAggregate (51)\n            +- Exchange (50)\n               +- HashAggregate (49)\n                  +- HashAggregate (48)\n                     +- ShuffleQueryStage (47)\n                        +- Exchange (46)\n                           +- * HashAggregate (45)\n                              +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (44)\n+- == Initial Plan ==\n   Project (89)\n   +- BroadcastNestedLoopJoin Cross BuildRight (88)\n      :- BroadcastNestedLoopJoin Cross BuildRight (81)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (71)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (64)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (60)\n      :  :  :  :  :- HashAggregate (56)\n      :  :  :  :  :  +- Exchange (55)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (59)\n      :  :  :  :     +- HashAggregate (58)\n      :  :  :  :        +- Exchange (57)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (63)\n      :  :  :     +- HashAggregate (62)\n      :  :  :        +- Exchange (61)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (70)\n      :  :     +- HashAggregate (69)\n      :  :        +- Exchange (68)\n      :  :           +- HashAggregate (67)\n      :  :              +- HashAggregate (66)\n      :  :                 +- Exchange (65)\n      :  :                    +- HashAggregate (21)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (20)\n      :  +- BroadcastExchange (80)\n      :     +- HashAggregate (79)\n      :        +- Exchange (78)\n      :           +- HashAggregate (77)\n      :              +- HashAggregate (76)\n      :                 +- Exchange (75)\n      :                    +- HashAggregate (74)\n      :                       +- HashAggregate (73)\n      :                          +- Exchange (72)\n      :                             +- HashAggregate (32)\n      :                                +- Project (31)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (30)\n      +- BroadcastExchange (87)\n         +- HashAggregate (86)\n            +- Exchange (85)\n               +- HashAggregate (84)\n                  +- HashAggregate (83)\n                     +- Exchange (82)\n                        +- HashAggregate (45)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (44)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1807]\n\n(19) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(20) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(21) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(22) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(23) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(24) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(25) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(26) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1830]\n\n(27) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(28) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1832]\n\n(29) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(30) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(31) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(32) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(33) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(34) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(35) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(36) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(37) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]\n\n(38) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(39) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(40) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]\n\n(41) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(42) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1868]\n\n(43) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(44) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(45) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(46) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(47) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(48) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(49) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(50) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]\n\n(51) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(52) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1893]\n\n(53) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(54) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(55) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(56) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(57) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(58) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(59) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(60) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(61) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(62) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(63) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(64) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(65) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(66) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(67) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(68) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(69) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(70) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(71) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(72) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(73) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(74) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(75) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(76) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(77) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(78) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(79) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(80) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(81) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(82) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(83) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(84) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(85) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(86) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(87) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(88) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(89) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(90) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1807]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3012,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3013,"metricType":"timing"},{"name":"peak memory","accumulatorId":3011,"metricType":"size"},{"name":"number of output rows","accumulatorId":3010,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3015,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3014,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3009,"metricType":"timing"},{"name":"time to build","accumulatorId":3008,"metricType":"timing"},{"name":"time to collect","accumulatorId":3007,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3006,"metricType":"sum"},{"name":"data size","accumulatorId":3005,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2940,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1832]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1830]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3080,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3081,"metricType":"timing"},{"name":"peak memory","accumulatorId":3079,"metricType":"size"},{"name":"number of output rows","accumulatorId":3078,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3083,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3082,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3074,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3075,"metricType":"timing"},{"name":"peak memory","accumulatorId":3073,"metricType":"size"},{"name":"number of output rows","accumulatorId":3072,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3077,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3076,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3070,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3064,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3071,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3065,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3062,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3059,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3068,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3061,"metricType":"sum"},{"name":"records read","accumulatorId":3058,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3056,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3057,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3054,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3060,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3053,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3063,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3052,"metricType":"sum"},{"name":"data size","accumulatorId":3050,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3066,"metricType":"size"},{"name":"number of partitions","accumulatorId":3051,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3067,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3055,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3069,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3046,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3047,"metricType":"timing"},{"name":"peak memory","accumulatorId":3045,"metricType":"size"},{"name":"number of output rows","accumulatorId":3044,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3049,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3048,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3043,"metricType":"timing"},{"name":"time to build","accumulatorId":3042,"metricType":"timing"},{"name":"time to collect","accumulatorId":3041,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3040,"metricType":"sum"},{"name":"data size","accumulatorId":3039,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2939,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1868]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3182,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3183,"metricType":"timing"},{"name":"peak memory","accumulatorId":3181,"metricType":"size"},{"name":"number of output rows","accumulatorId":3180,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3185,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3184,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3176,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3177,"metricType":"timing"},{"name":"peak memory","accumulatorId":3175,"metricType":"size"},{"name":"number of output rows","accumulatorId":3174,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3179,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3178,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3172,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3166,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3173,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3167,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3164,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3161,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3170,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3163,"metricType":"sum"},{"name":"records read","accumulatorId":3160,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3158,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3159,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3156,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3162,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3155,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3165,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3154,"metricType":"sum"},{"name":"data size","accumulatorId":3152,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3168,"metricType":"size"},{"name":"number of partitions","accumulatorId":3153,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3169,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3157,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3171,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3148,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3149,"metricType":"timing"},{"name":"peak memory","accumulatorId":3147,"metricType":"size"},{"name":"number of output rows","accumulatorId":3146,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3151,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3150,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3142,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3143,"metricType":"timing"},{"name":"peak memory","accumulatorId":3141,"metricType":"size"},{"name":"number of output rows","accumulatorId":3140,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3145,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3144,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3138,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3132,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3139,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3133,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3130,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3127,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3136,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3129,"metricType":"sum"},{"name":"records read","accumulatorId":3126,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3124,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3125,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3122,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3128,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3121,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3131,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3120,"metricType":"sum"},{"name":"data size","accumulatorId":3118,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3134,"metricType":"size"},{"name":"number of partitions","accumulatorId":3119,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3135,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3123,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3137,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3114,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3115,"metricType":"timing"},{"name":"peak memory","accumulatorId":3113,"metricType":"size"},{"name":"number of output rows","accumulatorId":3112,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3117,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3116,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3111,"metricType":"timing"},{"name":"time to build","accumulatorId":3110,"metricType":"timing"},{"name":"time to collect","accumulatorId":3109,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3108,"metricType":"sum"},{"name":"data size","accumulatorId":3107,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2938,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1893]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3250,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3251,"metricType":"timing"},{"name":"peak memory","accumulatorId":3249,"metricType":"size"},{"name":"number of output rows","accumulatorId":3248,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3253,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3252,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3244,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3245,"metricType":"timing"},{"name":"peak memory","accumulatorId":3243,"metricType":"size"},{"name":"number of output rows","accumulatorId":3242,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3247,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3246,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3231,"metricType":"sum"},{"name":"records read","accumulatorId":3228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3222,"metricType":"sum"},{"name":"data size","accumulatorId":3220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3236,"metricType":"size"},{"name":"number of partitions","accumulatorId":3221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3239,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3216,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3217,"metricType":"timing"},{"name":"peak memory","accumulatorId":3215,"metricType":"size"},{"name":"number of output rows","accumulatorId":3214,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3219,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3218,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3213,"metricType":"timing"},{"name":"time to build","accumulatorId":3212,"metricType":"timing"},{"name":"time to collect","accumulatorId":3211,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3210,"metricType":"sum"},{"name":"data size","accumulatorId":3209,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2937,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2949,1]]}
{"Event":"SparkListenerJobStart","Job ID":22,"Submission Time":1784293431704,"Stage Infos":[{"Stage ID":44,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[44],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":44,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431705,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":44,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431712,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2983,1]]}
{"Event":"SparkListenerJobStart","Job ID":23,"Submission Time":1784293431714,"Stage Infos":[{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"219\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[45],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"219\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431715,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":45,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431721,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3017,1]]}
{"Event":"SparkListenerJobStart","Job ID":24,"Submission Time":1784293431723,"Stage Infos":[{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"223\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[46],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"223\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431723,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431731,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3085,200]]}
{"Event":"SparkListenerJobStart","Job ID":25,"Submission Time":1784293431747,"Stage Infos":[{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[47],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431748,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":47,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431752,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3187,200]]}
{"Event":"SparkListenerJobStart","Job ID":26,"Submission Time":1784293431770,"Stage Infos":[{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[48],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431771,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskEnd","Stage ID":44,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431712,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431782,"Failed":false,"Killed":false,"Accumulables":[{"ID":2594,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2597,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2600,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2948,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2967,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2968,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2969,"Name":"shuffle write time","Update":"390513","Value":"390513","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2970,"Name":"duration","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3277,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":3278,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6743544,"Value":6743544,"Internal":true,"Count Failed Values":true},{"ID":3279,"Name":"internal.metrics.executorRunTime","Update":46,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":3280,"Name":"internal.metrics.executorCpuTime","Update":12388820,"Value":12388820,"Internal":true,"Count Failed Values":true},{"ID":3281,"Name":"internal.metrics.resultSize","Update":1879,"Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3305,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3306,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3307,"Name":"internal.metrics.shuffle.write.writeTime","Update":390513,"Value":390513,"Internal":true,"Count Failed Values":true},{"ID":3309,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":6743544,"Executor Run Time":46,"Executor CPU Time":12388820,"Peak Execution Memory":0,"Result Size":1879,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":390513,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":44,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431705,"Completion Time":1784293431785,"Accumulables":[{"ID":2594,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2597,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2600,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2948,"Name":"data size","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2967,"Name":"shuffle bytes written","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2968,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2969,"Name":"shuffle write time","Value":"390513","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2970,"Name":"duration","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3277,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":3278,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6743544,"Internal":true,"Count Failed Values":true},{"ID":3279,"Name":"internal.metrics.executorRunTime","Value":46,"Internal":true,"Count Failed Values":true},{"ID":3280,"Name":"internal.metrics.executorCpuTime","Value":12388820,"Internal":true,"Count Failed Values":true},{"ID":3281,"Name":"internal.metrics.resultSize","Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3305,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3306,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3307,"Name":"internal.metrics.shuffle.write.writeTime","Value":390513,"Internal":true,"Count Failed Values":true},{"ID":3309,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":22,"Completion Time":1784293431785,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431784,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431731,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431792,"Failed":false,"Killed":false,"Accumulables":[{"ID":2674,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2677,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3016,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3035,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3036,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3037,"Name":"shuffle write time","Update":"463964","Value":"463964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3038,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3347,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":3348,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4522368,"Value":4522368,"Internal":true,"Count Failed Values":true},{"ID":3349,"Name":"internal.metrics.executorRunTime","Update":47,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":3350,"Name":"internal.metrics.executorCpuTime","Update":11926686,"Value":11926686,"Internal":true,"Count Failed Values":true},{"ID":3351,"Name":"internal.metrics.resultSize","Update":1879,"Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3375,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3376,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3377,"Name":"internal.metrics.shuffle.write.writeTime","Update":463964,"Value":463964,"Internal":true,"Count Failed Values":true},{"ID":3379,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":4522368,"Executor Run Time":47,"Executor CPU Time":11926686,"Peak Execution Memory":0,"Result Size":1879,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":463964,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"223\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431723,"Completion Time":1784293431793,"Accumulables":[{"ID":2674,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2677,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2680,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3016,"Name":"data size","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3035,"Name":"shuffle bytes written","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3036,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3037,"Name":"shuffle write time","Value":"463964","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3038,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3347,"Name":"internal.metrics.executorDeserializeTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":3348,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4522368,"Internal":true,"Count Failed Values":true},{"ID":3349,"Name":"internal.metrics.executorRunTime","Value":47,"Internal":true,"Count Failed Values":true},{"ID":3350,"Name":"internal.metrics.executorCpuTime","Value":11926686,"Internal":true,"Count Failed Values":true},{"ID":3351,"Name":"internal.metrics.resultSize","Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3375,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3376,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3377,"Name":"internal.metrics.shuffle.write.writeTime","Value":463964,"Internal":true,"Count Failed Values":true},{"ID":3379,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":24,"Completion Time":1784293431794,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3255,200]]}
{"Event":"SparkListenerJobStart","Job ID":27,"Submission Time":1784293431797,"Stage Infos":[{"Stage ID":49,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"236\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[49],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":49,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"236\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431798,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":49,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431804,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (91)\n+- == Current Plan ==\n   Project (55)\n   +- BroadcastNestedLoopJoin Cross BuildRight (54)\n      :- BroadcastNestedLoopJoin Cross BuildRight (44)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (30)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (29)\n      :  :     +- HashAggregate (28)\n      :  :        +- Exchange (27)\n      :  :           +- HashAggregate (26)\n      :  :              +- HashAggregate (25)\n      :  :                 +- ShuffleQueryStage (24)\n      :  :                    +- Exchange (23)\n      :  :                       +- * HashAggregate (22)\n      :  :                          +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (43)\n      :     +- HashAggregate (42)\n      :        +- Exchange (41)\n      :           +- HashAggregate (40)\n      :              +- HashAggregate (39)\n      :                 +- Exchange (38)\n      :                    +- HashAggregate (37)\n      :                       +- HashAggregate (36)\n      :                          +- ShuffleQueryStage (35)\n      :                             +- Exchange (34)\n      :                                +- * HashAggregate (33)\n      :                                   +- * Project (32)\n      :                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (31)\n      +- BroadcastExchange (53)\n         +- HashAggregate (52)\n            +- Exchange (51)\n               +- HashAggregate (50)\n                  +- HashAggregate (49)\n                     +- ShuffleQueryStage (48)\n                        +- Exchange (47)\n                           +- * HashAggregate (46)\n                              +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (45)\n+- == Initial Plan ==\n   Project (90)\n   +- BroadcastNestedLoopJoin Cross BuildRight (89)\n      :- BroadcastNestedLoopJoin Cross BuildRight (82)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (72)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (65)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (61)\n      :  :  :  :  :- HashAggregate (57)\n      :  :  :  :  :  +- Exchange (56)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (60)\n      :  :  :  :     +- HashAggregate (59)\n      :  :  :  :        +- Exchange (58)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (64)\n      :  :  :     +- HashAggregate (63)\n      :  :  :        +- Exchange (62)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (71)\n      :  :     +- HashAggregate (70)\n      :  :        +- Exchange (69)\n      :  :           +- HashAggregate (68)\n      :  :              +- HashAggregate (67)\n      :  :                 +- Exchange (66)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (81)\n      :     +- HashAggregate (80)\n      :        +- Exchange (79)\n      :           +- HashAggregate (78)\n      :              +- HashAggregate (77)\n      :                 +- Exchange (76)\n      :                    +- HashAggregate (75)\n      :                       +- HashAggregate (74)\n      :                          +- Exchange (73)\n      :                             +- HashAggregate (33)\n      :                                +- Project (32)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (31)\n      +- BroadcastExchange (88)\n         +- HashAggregate (87)\n            +- Exchange (86)\n               +- HashAggregate (85)\n                  +- HashAggregate (84)\n                     +- Exchange (83)\n                        +- HashAggregate (46)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (45)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(26) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(27) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1830]\n\n(28) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(29) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1832]\n\n(30) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(31) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(32) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(33) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(34) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(35) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(36) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(37) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(38) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]\n\n(39) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(40) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(41) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]\n\n(42) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(43) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1868]\n\n(44) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(45) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(46) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(47) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(48) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(49) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(50) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(51) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]\n\n(52) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(53) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1893]\n\n(54) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(55) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(56) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(57) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(58) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(59) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(60) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(61) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(62) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(63) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(64) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(65) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(66) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(67) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(68) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(69) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(70) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(71) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(72) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(73) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(74) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(75) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(76) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(77) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(78) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(79) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(80) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(81) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(82) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(83) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(84) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(85) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(86) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(87) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(88) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(89) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(90) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(91) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1832]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1830]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3080,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3081,"metricType":"timing"},{"name":"peak memory","accumulatorId":3079,"metricType":"size"},{"name":"number of output rows","accumulatorId":3078,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3083,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3082,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3074,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3075,"metricType":"timing"},{"name":"peak memory","accumulatorId":3073,"metricType":"size"},{"name":"number of output rows","accumulatorId":3072,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3077,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3076,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3070,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3064,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3071,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3065,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3062,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3059,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3068,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3061,"metricType":"sum"},{"name":"records read","accumulatorId":3058,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3056,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3057,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3054,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3060,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3053,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3063,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3052,"metricType":"sum"},{"name":"data size","accumulatorId":3050,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3066,"metricType":"size"},{"name":"number of partitions","accumulatorId":3051,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3067,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3055,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3069,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3046,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3047,"metricType":"timing"},{"name":"peak memory","accumulatorId":3045,"metricType":"size"},{"name":"number of output rows","accumulatorId":3044,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3049,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3048,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3043,"metricType":"timing"},{"name":"time to build","accumulatorId":3042,"metricType":"timing"},{"name":"time to collect","accumulatorId":3041,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3040,"metricType":"sum"},{"name":"data size","accumulatorId":3039,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3489,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1868]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3182,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3183,"metricType":"timing"},{"name":"peak memory","accumulatorId":3181,"metricType":"size"},{"name":"number of output rows","accumulatorId":3180,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3185,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3184,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3176,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3177,"metricType":"timing"},{"name":"peak memory","accumulatorId":3175,"metricType":"size"},{"name":"number of output rows","accumulatorId":3174,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3179,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3178,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3172,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3166,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3173,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3167,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3164,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3161,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3170,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3163,"metricType":"sum"},{"name":"records read","accumulatorId":3160,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3158,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3159,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3156,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3162,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3155,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3165,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3154,"metricType":"sum"},{"name":"data size","accumulatorId":3152,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3168,"metricType":"size"},{"name":"number of partitions","accumulatorId":3153,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3169,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3157,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3171,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3148,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3149,"metricType":"timing"},{"name":"peak memory","accumulatorId":3147,"metricType":"size"},{"name":"number of output rows","accumulatorId":3146,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3151,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3150,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3142,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3143,"metricType":"timing"},{"name":"peak memory","accumulatorId":3141,"metricType":"size"},{"name":"number of output rows","accumulatorId":3140,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3145,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3144,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3138,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3132,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3139,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3133,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3130,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3127,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3136,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3129,"metricType":"sum"},{"name":"records read","accumulatorId":3126,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3124,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3125,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3122,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3128,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3121,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3131,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3120,"metricType":"sum"},{"name":"data size","accumulatorId":3118,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3134,"metricType":"size"},{"name":"number of partitions","accumulatorId":3119,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3135,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3123,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3137,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3114,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3115,"metricType":"timing"},{"name":"peak memory","accumulatorId":3113,"metricType":"size"},{"name":"number of output rows","accumulatorId":3112,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3117,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3116,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3111,"metricType":"timing"},{"name":"time to build","accumulatorId":3110,"metricType":"timing"},{"name":"time to collect","accumulatorId":3109,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3108,"metricType":"sum"},{"name":"data size","accumulatorId":3107,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3488,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1893]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3250,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3251,"metricType":"timing"},{"name":"peak memory","accumulatorId":3249,"metricType":"size"},{"name":"number of output rows","accumulatorId":3248,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3253,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3252,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3244,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3245,"metricType":"timing"},{"name":"peak memory","accumulatorId":3243,"metricType":"size"},{"name":"number of output rows","accumulatorId":3242,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3247,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3246,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3231,"metricType":"sum"},{"name":"records read","accumulatorId":3228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3222,"metricType":"sum"},{"name":"data size","accumulatorId":3220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3236,"metricType":"size"},{"name":"number of partitions","accumulatorId":3221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3239,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3216,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3217,"metricType":"timing"},{"name":"peak memory","accumulatorId":3215,"metricType":"size"},{"name":"number of output rows","accumulatorId":3214,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3219,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3218,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3213,"metricType":"timing"},{"name":"time to build","accumulatorId":3212,"metricType":"timing"},{"name":"time to collect","accumulatorId":3211,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3210,"metricType":"sum"},{"name":"data size","accumulatorId":3209,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3487,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":28,"Submission Time":1784293431826,"Stage Infos":[{"Stage ID":51,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"244\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[50],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":50,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"223\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[51,50],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"239\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId bbf83e5e-9bbc-4d91-bd15-568e10ce122d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":51,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"244\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[50],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431827,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"239\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId bbf83e5e-9bbc-4d91-bd15-568e10ce122d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":51,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431840,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":47,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431752,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431842,"Failed":false,"Killed":false,"Accumulables":[{"ID":2748,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2749,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2751,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2754,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3084,"Name":"data size","Update":"176","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3103,"Name":"shuffle bytes written","Update":"649","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3104,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3105,"Name":"shuffle write time","Update":"3885035","Value":"3885035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3106,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3382,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":3383,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5051260,"Value":5051260,"Internal":true,"Count Failed Values":true},{"ID":3384,"Name":"internal.metrics.executorRunTime","Update":66,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":3385,"Name":"internal.metrics.executorCpuTime","Update":31460994,"Value":31460994,"Internal":true,"Count Failed Values":true},{"ID":3386,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3391,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3410,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":649,"Value":649,"Internal":true,"Count Failed Values":true},{"ID":3411,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":3412,"Name":"internal.metrics.shuffle.write.writeTime","Update":3885035,"Value":3885035,"Internal":true,"Count Failed Values":true},{"ID":3414,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":5051260,"Executor Run Time":66,"Executor CPU Time":31460994,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":649,"Shuffle Write Time":3885035,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431748,"Completion Time":1784293431843,"Accumulables":[{"ID":2748,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2749,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2751,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2754,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3084,"Name":"data size","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3103,"Name":"shuffle bytes written","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3104,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3105,"Name":"shuffle write time","Value":"3885035","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3106,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3382,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":3383,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5051260,"Internal":true,"Count Failed Values":true},{"ID":3384,"Name":"internal.metrics.executorRunTime","Value":66,"Internal":true,"Count Failed Values":true},{"ID":3385,"Name":"internal.metrics.executorCpuTime","Value":31460994,"Internal":true,"Count Failed Values":true},{"ID":3386,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3391,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3410,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":649,"Internal":true,"Count Failed Values":true},{"ID":3411,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3412,"Name":"internal.metrics.shuffle.write.writeTime","Value":3885035,"Internal":true,"Count Failed Values":true},{"ID":3414,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":25,"Completion Time":1784293431847,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (93)\n+- == Current Plan ==\n   Project (57)\n   +- BroadcastNestedLoopJoin Cross BuildRight (56)\n      :- BroadcastNestedLoopJoin Cross BuildRight (46)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (32)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (31)\n      :  :     +- HashAggregate (30)\n      :  :        +- ShuffleQueryStage (29)\n      :  :           +- Exchange (28)\n      :  :              +- * HashAggregate (27)\n      :  :                 +- * HashAggregate (26)\n      :  :                    +- AQEShuffleRead (25)\n      :  :                       +- ShuffleQueryStage (24), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                          +- Exchange (23)\n      :  :                             +- * HashAggregate (22)\n      :  :                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (45)\n      :     +- HashAggregate (44)\n      :        +- Exchange (43)\n      :           +- HashAggregate (42)\n      :              +- HashAggregate (41)\n      :                 +- Exchange (40)\n      :                    +- HashAggregate (39)\n      :                       +- HashAggregate (38)\n      :                          +- ShuffleQueryStage (37)\n      :                             +- Exchange (36)\n      :                                +- * HashAggregate (35)\n      :                                   +- * Project (34)\n      :                                      +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (55)\n         +- HashAggregate (54)\n            +- Exchange (53)\n               +- HashAggregate (52)\n                  +- HashAggregate (51)\n                     +- ShuffleQueryStage (50)\n                        +- Exchange (49)\n                           +- * HashAggregate (48)\n                              +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (47)\n+- == Initial Plan ==\n   Project (92)\n   +- BroadcastNestedLoopJoin Cross BuildRight (91)\n      :- BroadcastNestedLoopJoin Cross BuildRight (84)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (74)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (67)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (63)\n      :  :  :  :  :- HashAggregate (59)\n      :  :  :  :  :  +- Exchange (58)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (62)\n      :  :  :  :     +- HashAggregate (61)\n      :  :  :  :        +- Exchange (60)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (66)\n      :  :  :     +- HashAggregate (65)\n      :  :  :        +- Exchange (64)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (73)\n      :  :     +- HashAggregate (72)\n      :  :        +- Exchange (71)\n      :  :           +- HashAggregate (70)\n      :  :              +- HashAggregate (69)\n      :  :                 +- Exchange (68)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (83)\n      :     +- HashAggregate (82)\n      :        +- Exchange (81)\n      :           +- HashAggregate (80)\n      :              +- HashAggregate (79)\n      :                 +- Exchange (78)\n      :                    +- HashAggregate (77)\n      :                       +- HashAggregate (76)\n      :                          +- Exchange (75)\n      :                             +- HashAggregate (35)\n      :                                +- Project (34)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (90)\n         +- HashAggregate (89)\n            +- Exchange (88)\n               +- HashAggregate (87)\n                  +- HashAggregate (86)\n                     +- Exchange (85)\n                        +- HashAggregate (48)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (47)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(28) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(29) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(30) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(31) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2080]\n\n(32) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(33) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(34) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(35) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(36) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(37) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(38) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(39) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(40) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]\n\n(41) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(42) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(43) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]\n\n(44) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(45) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1868]\n\n(46) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(47) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(48) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(49) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(50) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(51) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(52) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(53) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]\n\n(54) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(55) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1893]\n\n(56) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(57) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(58) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(59) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(60) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(61) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(62) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(63) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(64) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(65) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(66) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(67) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(68) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(69) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(70) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(71) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(72) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(73) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(74) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(75) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(76) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(77) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(78) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(79) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(80) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(81) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(82) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(83) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(84) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(85) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(86) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(87) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(88) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(89) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(90) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(91) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(92) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(93) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2080]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3548,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3549,"metricType":"timing"},{"name":"peak memory","accumulatorId":3547,"metricType":"size"},{"name":"number of output rows","accumulatorId":3546,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3551,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3550,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3545,"metricType":"timing"},{"name":"time to build","accumulatorId":3544,"metricType":"timing"},{"name":"time to collect","accumulatorId":3543,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3542,"metricType":"sum"},{"name":"data size","accumulatorId":3541,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3540,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1868]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1866]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1863]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3182,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3183,"metricType":"timing"},{"name":"peak memory","accumulatorId":3181,"metricType":"size"},{"name":"number of output rows","accumulatorId":3180,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3185,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3184,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3176,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3177,"metricType":"timing"},{"name":"peak memory","accumulatorId":3175,"metricType":"size"},{"name":"number of output rows","accumulatorId":3174,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3179,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3178,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3172,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3166,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3173,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3167,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3164,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3161,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3170,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3163,"metricType":"sum"},{"name":"records read","accumulatorId":3160,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3158,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3159,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3156,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3162,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3155,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3165,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3154,"metricType":"sum"},{"name":"data size","accumulatorId":3152,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3168,"metricType":"size"},{"name":"number of partitions","accumulatorId":3153,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3169,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3157,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3171,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3148,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3149,"metricType":"timing"},{"name":"peak memory","accumulatorId":3147,"metricType":"size"},{"name":"number of output rows","accumulatorId":3146,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3151,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3150,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3142,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3143,"metricType":"timing"},{"name":"peak memory","accumulatorId":3141,"metricType":"size"},{"name":"number of output rows","accumulatorId":3140,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3145,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3144,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3138,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3132,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3139,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3133,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3130,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3127,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3136,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3129,"metricType":"sum"},{"name":"records read","accumulatorId":3126,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3124,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3125,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3122,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3128,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3121,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3131,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3120,"metricType":"sum"},{"name":"data size","accumulatorId":3118,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3134,"metricType":"size"},{"name":"number of partitions","accumulatorId":3119,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3135,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3123,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3137,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3114,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3115,"metricType":"timing"},{"name":"peak memory","accumulatorId":3113,"metricType":"size"},{"name":"number of output rows","accumulatorId":3112,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3117,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3116,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3111,"metricType":"timing"},{"name":"time to build","accumulatorId":3110,"metricType":"timing"},{"name":"time to collect","accumulatorId":3109,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3108,"metricType":"sum"},{"name":"data size","accumulatorId":3107,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3539,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1893]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3250,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3251,"metricType":"timing"},{"name":"peak memory","accumulatorId":3249,"metricType":"size"},{"name":"number of output rows","accumulatorId":3248,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3253,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3252,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3244,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3245,"metricType":"timing"},{"name":"peak memory","accumulatorId":3243,"metricType":"size"},{"name":"number of output rows","accumulatorId":3242,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3247,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3246,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3231,"metricType":"sum"},{"name":"records read","accumulatorId":3228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3222,"metricType":"sum"},{"name":"data size","accumulatorId":3220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3236,"metricType":"size"},{"name":"number of partitions","accumulatorId":3221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3239,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3216,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3217,"metricType":"timing"},{"name":"peak memory","accumulatorId":3215,"metricType":"size"},{"name":"number of output rows","accumulatorId":3214,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3219,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3218,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3213,"metricType":"timing"},{"name":"time to build","accumulatorId":3212,"metricType":"timing"},{"name":"time to collect","accumulatorId":3211,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3210,"metricType":"sum"},{"name":"data size","accumulatorId":3209,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3538,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3587,1],[3589,1],[3588,660]]}
{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431784,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431882,"Failed":false,"Killed":false,"Accumulables":[{"ID":2856,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2857,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2859,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2862,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3186,"Name":"data size","Update":"432","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3205,"Name":"shuffle bytes written","Update":"1131","Value":"1131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3206,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3207,"Name":"shuffle write time","Update":"5046272","Value":"5046272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3208,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3417,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":3418,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6453118,"Value":6453118,"Internal":true,"Count Failed Values":true},{"ID":3419,"Name":"internal.metrics.executorRunTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":3420,"Name":"internal.metrics.executorCpuTime","Update":31813138,"Value":31813138,"Internal":true,"Count Failed Values":true},{"ID":3421,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3426,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3445,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1131,"Value":1131,"Internal":true,"Count Failed Values":true},{"ID":3446,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":3447,"Name":"internal.metrics.shuffle.write.writeTime","Update":5046272,"Value":5046272,"Internal":true,"Count Failed Values":true},{"ID":3449,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":6453118,"Executor Run Time":75,"Executor CPU Time":31813138,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1131,"Shuffle Write Time":5046272,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431771,"Completion Time":1784293431884,"Accumulables":[{"ID":2856,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2857,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2859,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2862,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3186,"Name":"data size","Value":"432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3205,"Name":"shuffle bytes written","Value":"1131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3206,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3207,"Name":"shuffle write time","Value":"5046272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3208,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3417,"Name":"internal.metrics.executorDeserializeTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":3418,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6453118,"Internal":true,"Count Failed Values":true},{"ID":3419,"Name":"internal.metrics.executorRunTime","Value":75,"Internal":true,"Count Failed Values":true},{"ID":3420,"Name":"internal.metrics.executorCpuTime","Value":31813138,"Internal":true,"Count Failed Values":true},{"ID":3421,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3426,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3445,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1131,"Internal":true,"Count Failed Values":true},{"ID":3446,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":3447,"Name":"internal.metrics.shuffle.write.writeTime","Value":5046272,"Internal":true,"Count Failed Values":true},{"ID":3449,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":26,"Completion Time":1784293431884,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":51,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431840,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431882,"Failed":false,"Killed":false,"Accumulables":[{"ID":3019,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3022,"Name":"local bytes read","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3023,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3024,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3496,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3497,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3500,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3503,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":3504,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5031200,"Value":5031200,"Internal":true,"Count Failed Values":true},{"ID":3505,"Name":"internal.metrics.executorRunTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":3506,"Name":"internal.metrics.executorCpuTime","Update":8971449,"Value":8971449,"Internal":true,"Count Failed Values":true},{"ID":3507,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3514,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3515,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3516,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3517,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3518,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3519,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3520,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3521,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3522,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3524,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3525,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3526,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3527,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3528,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3529,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3530,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":5031200,"Executor Run Time":12,"Executor CPU Time":8971449,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":59,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":51,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":89,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"239\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"244\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[50],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431827,"Completion Time":1784293431885,"Accumulables":[{"ID":3019,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3022,"Name":"local bytes read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3023,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3024,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3496,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3497,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3500,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3503,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":3504,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5031200,"Internal":true,"Count Failed Values":true},{"ID":3505,"Name":"internal.metrics.executorRunTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":3506,"Name":"internal.metrics.executorCpuTime","Value":8971449,"Internal":true,"Count Failed Values":true},{"ID":3507,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3514,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3515,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3516,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3517,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3518,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3519,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3520,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3521,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3522,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3524,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3525,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3526,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3527,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3528,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3529,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3530,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":28,"Completion Time":1784293431885,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3553,1]]}
{"Event":"SparkListenerJobStart","Job ID":29,"Submission Time":1784293431887,"Stage Infos":[{"Stage ID":52,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"250\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[52,53],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"250\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431894,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3495,8],[3494,1],[3493,77],[3492,1],[3491,16]]}
{"Event":"SparkListenerTaskEnd","Stage ID":49,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431804,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431895,"Failed":false,"Killed":false,"Accumulables":[{"ID":2930,"Name":"number of output rows","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2931,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2933,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2936,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3254,"Name":"data size","Update":"360","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3273,"Name":"shuffle bytes written","Update":"891","Value":"891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3274,"Name":"shuffle records written","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3275,"Name":"shuffle write time","Update":"3633158","Value":"3633158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3276,"Name":"duration","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3452,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":3453,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6200166,"Value":6200166,"Internal":true,"Count Failed Values":true},{"ID":3454,"Name":"internal.metrics.executorRunTime","Update":67,"Value":67,"Internal":true,"Count Failed Values":true},{"ID":3455,"Name":"internal.metrics.executorCpuTime","Update":29171955,"Value":29171955,"Internal":true,"Count Failed Values":true},{"ID":3456,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3461,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3480,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":891,"Value":891,"Internal":true,"Count Failed Values":true},{"ID":3481,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":3482,"Name":"internal.metrics.shuffle.write.writeTime","Update":3633158,"Value":3633158,"Internal":true,"Count Failed Values":true},{"ID":3484,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":6200166,"Executor Run Time":67,"Executor CPU Time":29171955,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":891,"Shuffle Write Time":3633158,"Shuffle Records Written":15},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":49,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"236\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431798,"Completion Time":1784293431901,"Accumulables":[{"ID":2930,"Name":"number of output rows","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2931,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2933,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2936,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3254,"Name":"data size","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3273,"Name":"shuffle bytes written","Value":"891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3274,"Name":"shuffle records written","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3275,"Name":"shuffle write time","Value":"3633158","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3276,"Name":"duration","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3452,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":3453,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6200166,"Internal":true,"Count Failed Values":true},{"ID":3454,"Name":"internal.metrics.executorRunTime","Value":67,"Internal":true,"Count Failed Values":true},{"ID":3455,"Name":"internal.metrics.executorCpuTime","Value":29171955,"Internal":true,"Count Failed Values":true},{"ID":3456,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3461,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3480,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":891,"Internal":true,"Count Failed Values":true},{"ID":3481,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":15,"Internal":true,"Count Failed Values":true},{"ID":3482,"Name":"internal.metrics.shuffle.write.writeTime","Value":3633158,"Internal":true,"Count Failed Values":true},{"ID":3484,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":27,"Completion Time":1784293431902,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskStart","Stage ID":53,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431900,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (95)\n+- == Current Plan ==\n   Project (59)\n   +- BroadcastNestedLoopJoin Cross BuildRight (58)\n      :- BroadcastNestedLoopJoin Cross BuildRight (48)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (32)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (31)\n      :  :     +- HashAggregate (30)\n      :  :        +- ShuffleQueryStage (29)\n      :  :           +- Exchange (28)\n      :  :              +- * HashAggregate (27)\n      :  :                 +- * HashAggregate (26)\n      :  :                    +- AQEShuffleRead (25)\n      :  :                       +- ShuffleQueryStage (24), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                          +- Exchange (23)\n      :  :                             +- * HashAggregate (22)\n      :  :                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (47)\n      :     +- HashAggregate (46)\n      :        +- Exchange (45)\n      :           +- HashAggregate (44)\n      :              +- HashAggregate (43)\n      :                 +- ShuffleQueryStage (42)\n      :                    +- Exchange (41)\n      :                       +- * HashAggregate (40)\n      :                          +- * HashAggregate (39)\n      :                             +- AQEShuffleRead (38)\n      :                                +- ShuffleQueryStage (37), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                   +- Exchange (36)\n      :                                      +- * HashAggregate (35)\n      :                                         +- * Project (34)\n      :                                            +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (57)\n         +- HashAggregate (56)\n            +- Exchange (55)\n               +- HashAggregate (54)\n                  +- HashAggregate (53)\n                     +- ShuffleQueryStage (52)\n                        +- Exchange (51)\n                           +- * HashAggregate (50)\n                              +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (49)\n+- == Initial Plan ==\n   Project (94)\n   +- BroadcastNestedLoopJoin Cross BuildRight (93)\n      :- BroadcastNestedLoopJoin Cross BuildRight (86)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (76)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (69)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (65)\n      :  :  :  :  :- HashAggregate (61)\n      :  :  :  :  :  +- Exchange (60)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (64)\n      :  :  :  :     +- HashAggregate (63)\n      :  :  :  :        +- Exchange (62)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (68)\n      :  :  :     +- HashAggregate (67)\n      :  :  :        +- Exchange (66)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (75)\n      :  :     +- HashAggregate (74)\n      :  :        +- Exchange (73)\n      :  :           +- HashAggregate (72)\n      :  :              +- HashAggregate (71)\n      :  :                 +- Exchange (70)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (85)\n      :     +- HashAggregate (84)\n      :        +- Exchange (83)\n      :           +- HashAggregate (82)\n      :              +- HashAggregate (81)\n      :                 +- Exchange (80)\n      :                    +- HashAggregate (79)\n      :                       +- HashAggregate (78)\n      :                          +- Exchange (77)\n      :                             +- HashAggregate (35)\n      :                                +- Project (34)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (92)\n         +- HashAggregate (91)\n            +- Exchange (90)\n               +- HashAggregate (89)\n                  +- HashAggregate (88)\n                     +- Exchange (87)\n                        +- HashAggregate (50)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (49)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(28) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(29) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(30) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(31) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2080]\n\n(32) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(33) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(34) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(35) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(36) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(37) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(38) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(39) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(41) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(42) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(43) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(44) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(45) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]\n\n(46) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(47) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2167]\n\n(48) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(49) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(50) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(51) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(52) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(53) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(54) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(55) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]\n\n(56) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(57) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1893]\n\n(58) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(59) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(60) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(61) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(62) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(63) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(64) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(65) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(66) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(67) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(68) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(69) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(70) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(71) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(72) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(73) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(74) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(75) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(76) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(77) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(78) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(79) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(80) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(81) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(82) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(83) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(84) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(85) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(86) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(87) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(88) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(89) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(90) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(91) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(92) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(93) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(94) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(95) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2080]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3548,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3549,"metricType":"timing"},{"name":"peak memory","accumulatorId":3547,"metricType":"size"},{"name":"number of output rows","accumulatorId":3546,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3551,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3550,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3545,"metricType":"timing"},{"name":"time to build","accumulatorId":3544,"metricType":"timing"},{"name":"time to collect","accumulatorId":3543,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3542,"metricType":"sum"},{"name":"data size","accumulatorId":3541,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3540,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2167]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3668,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3669,"metricType":"timing"},{"name":"peak memory","accumulatorId":3667,"metricType":"size"},{"name":"number of output rows","accumulatorId":3666,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3671,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3670,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3662,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3663,"metricType":"timing"},{"name":"peak memory","accumulatorId":3661,"metricType":"size"},{"name":"number of output rows","accumulatorId":3660,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3665,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3664,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3658,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3652,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3659,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3653,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3650,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3647,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3656,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3649,"metricType":"sum"},{"name":"records read","accumulatorId":3646,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3644,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3645,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3642,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3648,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3641,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3651,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3640,"metricType":"sum"},{"name":"data size","accumulatorId":3638,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3654,"metricType":"size"},{"name":"number of partitions","accumulatorId":3639,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3655,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3643,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3657,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3634,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3635,"metricType":"timing"},{"name":"peak memory","accumulatorId":3633,"metricType":"size"},{"name":"number of output rows","accumulatorId":3632,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3637,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3636,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3631,"metricType":"timing"},{"name":"time to build","accumulatorId":3630,"metricType":"timing"},{"name":"time to collect","accumulatorId":3629,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3628,"metricType":"sum"},{"name":"data size","accumulatorId":3627,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3626,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1893]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1891]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3250,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3251,"metricType":"timing"},{"name":"peak memory","accumulatorId":3249,"metricType":"size"},{"name":"number of output rows","accumulatorId":3248,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3253,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3252,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3244,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3245,"metricType":"timing"},{"name":"peak memory","accumulatorId":3243,"metricType":"size"},{"name":"number of output rows","accumulatorId":3242,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3247,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3246,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3231,"metricType":"sum"},{"name":"records read","accumulatorId":3228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3222,"metricType":"sum"},{"name":"data size","accumulatorId":3220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3236,"metricType":"size"},{"name":"number of partitions","accumulatorId":3221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3239,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3216,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3217,"metricType":"timing"},{"name":"peak memory","accumulatorId":3215,"metricType":"size"},{"name":"number of output rows","accumulatorId":3214,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3219,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3218,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3213,"metricType":"timing"},{"name":"time to build","accumulatorId":3212,"metricType":"timing"},{"name":"time to collect","accumulatorId":3211,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3210,"metricType":"sum"},{"name":"data size","accumulatorId":3209,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3625,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3707,1],[3709,1],[3708,1182]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3673,200]]}
{"Event":"SparkListenerJobStart","Job ID":30,"Submission Time":1784293431930,"Stage Infos":[{"Stage ID":54,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"262\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"258\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[54,55],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"262\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"258\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431932,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":55,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431937,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":53,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431900,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431940,"Failed":false,"Killed":false,"Accumulables":[{"ID":3087,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3090,"Name":"local bytes read","Update":"649","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3091,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3092,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3552,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3571,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3572,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3573,"Name":"shuffle write time","Update":"382935","Value":"382935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3574,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3575,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3578,"Name":"time in aggregation build","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3581,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3582,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3584,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3590,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":3591,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7686590,"Value":7686590,"Internal":true,"Count Failed Values":true},{"ID":3592,"Name":"internal.metrics.executorRunTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":3593,"Name":"internal.metrics.executorCpuTime","Update":19514937,"Value":19514937,"Internal":true,"Count Failed Values":true},{"ID":3594,"Name":"internal.metrics.resultSize","Update":5358,"Value":5358,"Internal":true,"Count Failed Values":true},{"ID":3599,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3601,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3602,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3603,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3604,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3605,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":649,"Value":649,"Internal":true,"Count Failed Values":true},{"ID":3606,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3607,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":3608,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3609,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3610,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3611,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3612,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3613,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3614,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3615,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3616,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3617,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3618,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3619,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3620,"Name":"internal.metrics.shuffle.write.writeTime","Update":382935,"Value":382935,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":7686590,"Executor Run Time":21,"Executor CPU Time":19514937,"Peak Execution Memory":262144,"Result Size":5358,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":649,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":382935,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"250\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[52],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431894,"Completion Time":1784293431941,"Accumulables":[{"ID":3087,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3090,"Name":"local bytes read","Value":"649","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3091,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3092,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3552,"Name":"data size","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3571,"Name":"shuffle bytes written","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3572,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3573,"Name":"shuffle write time","Value":"382935","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3574,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3575,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3578,"Name":"time in aggregation build","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3581,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3582,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3584,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3590,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":3591,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7686590,"Internal":true,"Count Failed Values":true},{"ID":3592,"Name":"internal.metrics.executorRunTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":3593,"Name":"internal.metrics.executorCpuTime","Value":19514937,"Internal":true,"Count Failed Values":true},{"ID":3594,"Name":"internal.metrics.resultSize","Value":5358,"Internal":true,"Count Failed Values":true},{"ID":3599,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3601,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3602,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3603,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3604,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3605,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":649,"Internal":true,"Count Failed Values":true},{"ID":3606,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3607,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3608,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3609,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3610,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3611,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3612,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3613,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3614,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3615,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3616,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3617,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3618,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3619,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3620,"Name":"internal.metrics.shuffle.write.writeTime","Value":382935,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":29,"Completion Time":1784293431942,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (97)\n+- == Current Plan ==\n   Project (61)\n   +- BroadcastNestedLoopJoin Cross BuildRight (60)\n      :- BroadcastNestedLoopJoin Cross BuildRight (48)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (32)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (31)\n      :  :     +- HashAggregate (30)\n      :  :        +- ShuffleQueryStage (29)\n      :  :           +- Exchange (28)\n      :  :              +- * HashAggregate (27)\n      :  :                 +- * HashAggregate (26)\n      :  :                    +- AQEShuffleRead (25)\n      :  :                       +- ShuffleQueryStage (24), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                          +- Exchange (23)\n      :  :                             +- * HashAggregate (22)\n      :  :                                +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (47)\n      :     +- HashAggregate (46)\n      :        +- Exchange (45)\n      :           +- HashAggregate (44)\n      :              +- HashAggregate (43)\n      :                 +- ShuffleQueryStage (42)\n      :                    +- Exchange (41)\n      :                       +- * HashAggregate (40)\n      :                          +- * HashAggregate (39)\n      :                             +- AQEShuffleRead (38)\n      :                                +- ShuffleQueryStage (37), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                   +- Exchange (36)\n      :                                      +- * HashAggregate (35)\n      :                                         +- * Project (34)\n      :                                            +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (59)\n         +- HashAggregate (58)\n            +- ShuffleQueryStage (57)\n               +- Exchange (56)\n                  +- * HashAggregate (55)\n                     +- * HashAggregate (54)\n                        +- AQEShuffleRead (53)\n                           +- ShuffleQueryStage (52), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                              +- Exchange (51)\n                                 +- * HashAggregate (50)\n                                    +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (49)\n+- == Initial Plan ==\n   Project (96)\n   +- BroadcastNestedLoopJoin Cross BuildRight (95)\n      :- BroadcastNestedLoopJoin Cross BuildRight (88)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (78)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (71)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (67)\n      :  :  :  :  :- HashAggregate (63)\n      :  :  :  :  :  +- Exchange (62)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (66)\n      :  :  :  :     +- HashAggregate (65)\n      :  :  :  :        +- Exchange (64)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (70)\n      :  :  :     +- HashAggregate (69)\n      :  :  :        +- Exchange (68)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (77)\n      :  :     +- HashAggregate (76)\n      :  :        +- Exchange (75)\n      :  :           +- HashAggregate (74)\n      :  :              +- HashAggregate (73)\n      :  :                 +- Exchange (72)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (87)\n      :     +- HashAggregate (86)\n      :        +- Exchange (85)\n      :           +- HashAggregate (84)\n      :              +- HashAggregate (83)\n      :                 +- Exchange (82)\n      :                    +- HashAggregate (81)\n      :                       +- HashAggregate (80)\n      :                          +- Exchange (79)\n      :                             +- HashAggregate (35)\n      :                                +- Project (34)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (33)\n      +- BroadcastExchange (94)\n         +- HashAggregate (93)\n            +- Exchange (92)\n               +- HashAggregate (91)\n                  +- HashAggregate (90)\n                     +- Exchange (89)\n                        +- HashAggregate (50)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (49)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(28) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(29) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(30) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(31) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2080]\n\n(32) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(33) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(34) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(35) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(36) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(37) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(38) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(39) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(41) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(42) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(43) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(44) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(45) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]\n\n(46) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(47) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2167]\n\n(48) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(49) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(50) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(51) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(52) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(53) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(54) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(55) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(56) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(57) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(58) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(59) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2254]\n\n(60) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(61) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(62) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(63) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(64) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(65) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(66) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(67) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(68) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(69) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(70) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(71) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(72) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(73) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(74) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(75) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(76) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(77) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(78) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(79) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(80) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(81) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(82) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(83) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(84) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(85) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(86) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(87) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(88) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(89) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(90) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(91) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(92) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(93) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(94) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(95) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(96) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(97) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2080]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3548,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3549,"metricType":"timing"},{"name":"peak memory","accumulatorId":3547,"metricType":"size"},{"name":"number of output rows","accumulatorId":3546,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3551,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3550,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3545,"metricType":"timing"},{"name":"time to build","accumulatorId":3544,"metricType":"timing"},{"name":"time to collect","accumulatorId":3543,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3542,"metricType":"sum"},{"name":"data size","accumulatorId":3541,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3540,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2167]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3668,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3669,"metricType":"timing"},{"name":"peak memory","accumulatorId":3667,"metricType":"size"},{"name":"number of output rows","accumulatorId":3666,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3671,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3670,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3662,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3663,"metricType":"timing"},{"name":"peak memory","accumulatorId":3661,"metricType":"size"},{"name":"number of output rows","accumulatorId":3660,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3665,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3664,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3658,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3652,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3659,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3653,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3650,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3647,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3656,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3649,"metricType":"sum"},{"name":"records read","accumulatorId":3646,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3644,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3645,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3642,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3648,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3641,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3651,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3640,"metricType":"sum"},{"name":"data size","accumulatorId":3638,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3654,"metricType":"size"},{"name":"number of partitions","accumulatorId":3639,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3655,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3643,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3657,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3634,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3635,"metricType":"timing"},{"name":"peak memory","accumulatorId":3633,"metricType":"size"},{"name":"number of output rows","accumulatorId":3632,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3637,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3636,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3631,"metricType":"timing"},{"name":"time to build","accumulatorId":3630,"metricType":"timing"},{"name":"time to collect","accumulatorId":3629,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3628,"metricType":"sum"},{"name":"data size","accumulatorId":3627,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3626,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2254]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3753,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3754,"metricType":"timing"},{"name":"peak memory","accumulatorId":3752,"metricType":"size"},{"name":"number of output rows","accumulatorId":3751,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3756,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3755,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3750,"metricType":"timing"},{"name":"time to build","accumulatorId":3749,"metricType":"timing"},{"name":"time to collect","accumulatorId":3748,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3747,"metricType":"sum"},{"name":"data size","accumulatorId":3746,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3745,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3792,1],[3794,1],[3793,930]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3758,1]]}
{"Event":"SparkListenerJobStart","Job ID":31,"Submission Time":1784293431965,"Stage Infos":[{"Stage ID":56,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"236\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"268\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"264\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[56,57],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"268\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"264\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431966,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":57,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431972,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (98)\n+- == Current Plan ==\n   Project (62)\n   +- BroadcastNestedLoopJoin Cross BuildRight (61)\n      :- BroadcastNestedLoopJoin Cross BuildRight (49)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (33)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastQueryStage (32)\n      :  :     +- BroadcastExchange (31)\n      :  :        +- * HashAggregate (30)\n      :  :           +- ShuffleQueryStage (29), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (28)\n      :  :                 +- * HashAggregate (27)\n      :  :                    +- * HashAggregate (26)\n      :  :                       +- AQEShuffleRead (25)\n      :  :                          +- ShuffleQueryStage (24), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (23)\n      :  :                                +- * HashAggregate (22)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (48)\n      :     +- HashAggregate (47)\n      :        +- Exchange (46)\n      :           +- HashAggregate (45)\n      :              +- HashAggregate (44)\n      :                 +- ShuffleQueryStage (43)\n      :                    +- Exchange (42)\n      :                       +- * HashAggregate (41)\n      :                          +- * HashAggregate (40)\n      :                             +- AQEShuffleRead (39)\n      :                                +- ShuffleQueryStage (38), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                   +- Exchange (37)\n      :                                      +- * HashAggregate (36)\n      :                                         +- * Project (35)\n      :                                            +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n      +- BroadcastExchange (60)\n         +- HashAggregate (59)\n            +- ShuffleQueryStage (58)\n               +- Exchange (57)\n                  +- * HashAggregate (56)\n                     +- * HashAggregate (55)\n                        +- AQEShuffleRead (54)\n                           +- ShuffleQueryStage (53), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                              +- Exchange (52)\n                                 +- * HashAggregate (51)\n                                    +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (50)\n+- == Initial Plan ==\n   Project (97)\n   +- BroadcastNestedLoopJoin Cross BuildRight (96)\n      :- BroadcastNestedLoopJoin Cross BuildRight (89)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (79)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (72)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (68)\n      :  :  :  :  :- HashAggregate (64)\n      :  :  :  :  :  +- Exchange (63)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (67)\n      :  :  :  :     +- HashAggregate (66)\n      :  :  :  :        +- Exchange (65)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (71)\n      :  :  :     +- HashAggregate (70)\n      :  :  :        +- Exchange (69)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (78)\n      :  :     +- HashAggregate (77)\n      :  :        +- Exchange (76)\n      :  :           +- HashAggregate (75)\n      :  :              +- HashAggregate (74)\n      :  :                 +- Exchange (73)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (88)\n      :     +- HashAggregate (87)\n      :        +- Exchange (86)\n      :           +- HashAggregate (85)\n      :              +- HashAggregate (84)\n      :                 +- Exchange (83)\n      :                    +- HashAggregate (82)\n      :                       +- HashAggregate (81)\n      :                          +- Exchange (80)\n      :                             +- HashAggregate (36)\n      :                                +- Project (35)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n      +- BroadcastExchange (95)\n         +- HashAggregate (94)\n            +- Exchange (93)\n               +- HashAggregate (92)\n                  +- HashAggregate (91)\n                     +- Exchange (90)\n                        +- HashAggregate (51)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (50)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(28) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(29) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(30) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(31) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(32) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(33) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(34) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(35) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(37) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(42) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(43) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(44) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(45) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(46) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]\n\n(47) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(48) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2167]\n\n(49) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(50) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(51) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(52) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(53) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(54) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(55) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(56) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(57) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(58) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(59) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(60) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2254]\n\n(61) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(62) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(63) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(64) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(65) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(66) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(67) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(68) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(69) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(70) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(71) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(72) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(73) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(74) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(75) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(76) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(77) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(78) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(79) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(80) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(81) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(82) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(83) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(84) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(85) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(86) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(87) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(88) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(89) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(90) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(91) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(92) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(93) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(94) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(95) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(96) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(97) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(98) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3832,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2167]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2165]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3668,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3669,"metricType":"timing"},{"name":"peak memory","accumulatorId":3667,"metricType":"size"},{"name":"number of output rows","accumulatorId":3666,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3671,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3670,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3662,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3663,"metricType":"timing"},{"name":"peak memory","accumulatorId":3661,"metricType":"size"},{"name":"number of output rows","accumulatorId":3660,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3665,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3664,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3658,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3652,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3659,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3653,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3650,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3647,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3656,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3649,"metricType":"sum"},{"name":"records read","accumulatorId":3646,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3644,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3645,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3642,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3648,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3641,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3651,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3640,"metricType":"sum"},{"name":"data size","accumulatorId":3638,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3654,"metricType":"size"},{"name":"number of partitions","accumulatorId":3639,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3655,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3643,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3657,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3634,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3635,"metricType":"timing"},{"name":"peak memory","accumulatorId":3633,"metricType":"size"},{"name":"number of output rows","accumulatorId":3632,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3637,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3636,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3631,"metricType":"timing"},{"name":"time to build","accumulatorId":3630,"metricType":"timing"},{"name":"time to collect","accumulatorId":3629,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3628,"metricType":"sum"},{"name":"data size","accumulatorId":3627,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3831,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2254]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3753,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3754,"metricType":"timing"},{"name":"peak memory","accumulatorId":3752,"metricType":"size"},{"name":"number of output rows","accumulatorId":3751,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3756,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3755,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3750,"metricType":"timing"},{"name":"time to build","accumulatorId":3749,"metricType":"timing"},{"name":"time to collect","accumulatorId":3748,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3747,"metricType":"sum"},{"name":"data size","accumulatorId":3746,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3830,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":55,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431937,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293431990,"Failed":false,"Killed":false,"Accumulables":[{"ID":3189,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3192,"Name":"local bytes read","Update":"1131","Value":"1131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3193,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3194,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3672,"Name":"data size","Update":"120","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3691,"Name":"shuffle bytes written","Update":"312","Value":"312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3692,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3693,"Name":"shuffle write time","Update":"2685206","Value":"2685206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3694,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3695,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3696,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3698,"Name":"time in aggregation build","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3701,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3702,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3704,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3710,"Name":"internal.metrics.executorDeserializeTime","Update":8,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":3711,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4797301,"Value":4797301,"Internal":true,"Count Failed Values":true},{"ID":3712,"Name":"internal.metrics.executorRunTime","Update":39,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":3713,"Name":"internal.metrics.executorCpuTime","Update":34795628,"Value":34795628,"Internal":true,"Count Failed Values":true},{"ID":3714,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":3719,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":3721,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3722,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3723,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3724,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3725,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1131,"Value":1131,"Internal":true,"Count Failed Values":true},{"ID":3726,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3727,"Name":"internal.metrics.shuffle.read.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":3728,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3729,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3730,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3731,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3732,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3733,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3734,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3735,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3736,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3737,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3738,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":312,"Value":312,"Internal":true,"Count Failed Values":true},{"ID":3739,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":3740,"Name":"internal.metrics.shuffle.write.writeTime","Update":2685206,"Value":2685206,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":8,"Executor Deserialize CPU Time":4797301,"Executor Run Time":39,"Executor CPU Time":34795628,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1131,"Total Records Read":18,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":312,"Shuffle Write Time":2685206,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"262\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"258\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[54],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431932,"Completion Time":1784293431990,"Accumulables":[{"ID":3189,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3192,"Name":"local bytes read","Value":"1131","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3193,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3194,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3672,"Name":"data size","Value":"120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3691,"Name":"shuffle bytes written","Value":"312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3692,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3693,"Name":"shuffle write time","Value":"2685206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3694,"Name":"duration","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3695,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3696,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3698,"Name":"time in aggregation build","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3701,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3702,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3704,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3710,"Name":"internal.metrics.executorDeserializeTime","Value":8,"Internal":true,"Count Failed Values":true},{"ID":3711,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4797301,"Internal":true,"Count Failed Values":true},{"ID":3712,"Name":"internal.metrics.executorRunTime","Value":39,"Internal":true,"Count Failed Values":true},{"ID":3713,"Name":"internal.metrics.executorCpuTime","Value":34795628,"Internal":true,"Count Failed Values":true},{"ID":3714,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":3719,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":3721,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3722,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3723,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3724,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3725,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1131,"Internal":true,"Count Failed Values":true},{"ID":3726,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3727,"Name":"internal.metrics.shuffle.read.recordsRead","Value":18,"Internal":true,"Count Failed Values":true},{"ID":3728,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3729,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3730,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3731,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3732,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3733,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3734,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3735,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3736,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3737,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3738,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":312,"Internal":true,"Count Failed Values":true},{"ID":3739,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":3740,"Name":"internal.metrics.shuffle.write.writeTime","Value":2685206,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":30,"Completion Time":1784293431991,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":32,"Submission Time":1784293431992,"Stage Infos":[{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":101,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"269\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"274\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"270\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":58,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"226\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":72,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[72],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":59,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":92,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"245\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"246\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"250\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[58],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[60,58,59],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"269\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 693bc474-3697-462b-be22-0950077670d0)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":101,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"269\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"274\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"270\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431993,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"269\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 693bc474-3697-462b-be22-0950077670d0)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":60,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431998,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (100)\n+- == Current Plan ==\n   Project (64)\n   +- BroadcastNestedLoopJoin Cross BuildRight (63)\n      :- BroadcastNestedLoopJoin Cross BuildRight (51)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (33)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (20)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (12)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (11)\n      :  :  :  :     +- HashAggregate (10)\n      :  :  :  :        +- ShuffleQueryStage (9)\n      :  :  :  :           +- Exchange (8)\n      :  :  :  :              +- * HashAggregate (7)\n      :  :  :  :                 +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (18)\n      :  :  :        +- * HashAggregate (17)\n      :  :  :           +- ShuffleQueryStage (16), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (15)\n      :  :  :                 +- * HashAggregate (14)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastQueryStage (32)\n      :  :     +- BroadcastExchange (31)\n      :  :        +- * HashAggregate (30)\n      :  :           +- ShuffleQueryStage (29), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (28)\n      :  :                 +- * HashAggregate (27)\n      :  :                    +- * HashAggregate (26)\n      :  :                       +- AQEShuffleRead (25)\n      :  :                          +- ShuffleQueryStage (24), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (23)\n      :  :                                +- * HashAggregate (22)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (50)\n      :     +- HashAggregate (49)\n      :        +- ShuffleQueryStage (48)\n      :           +- Exchange (47)\n      :              +- * HashAggregate (46)\n      :                 +- * HashAggregate (45)\n      :                    +- AQEShuffleRead (44)\n      :                       +- ShuffleQueryStage (43), Statistics(sizeInBytes=120.0 B, rowCount=5)\n      :                          +- Exchange (42)\n      :                             +- * HashAggregate (41)\n      :                                +- * HashAggregate (40)\n      :                                   +- AQEShuffleRead (39)\n      :                                      +- ShuffleQueryStage (38), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                         +- Exchange (37)\n      :                                            +- * HashAggregate (36)\n      :                                               +- * Project (35)\n      :                                                  +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n      +- BroadcastExchange (62)\n         +- HashAggregate (61)\n            +- ShuffleQueryStage (60)\n               +- Exchange (59)\n                  +- * HashAggregate (58)\n                     +- * HashAggregate (57)\n                        +- AQEShuffleRead (56)\n                           +- ShuffleQueryStage (55), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                              +- Exchange (54)\n                                 +- * HashAggregate (53)\n                                    +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (52)\n+- == Initial Plan ==\n   Project (99)\n   +- BroadcastNestedLoopJoin Cross BuildRight (98)\n      :- BroadcastNestedLoopJoin Cross BuildRight (91)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (81)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (74)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (70)\n      :  :  :  :  :- HashAggregate (66)\n      :  :  :  :  :  +- Exchange (65)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (69)\n      :  :  :  :     +- HashAggregate (68)\n      :  :  :  :        +- Exchange (67)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (73)\n      :  :  :     +- HashAggregate (72)\n      :  :  :        +- Exchange (71)\n      :  :  :           +- HashAggregate (14)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (13)\n      :  :  +- BroadcastExchange (80)\n      :  :     +- HashAggregate (79)\n      :  :        +- Exchange (78)\n      :  :           +- HashAggregate (77)\n      :  :              +- HashAggregate (76)\n      :  :                 +- Exchange (75)\n      :  :                    +- HashAggregate (22)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (21)\n      :  +- BroadcastExchange (90)\n      :     +- HashAggregate (89)\n      :        +- Exchange (88)\n      :           +- HashAggregate (87)\n      :              +- HashAggregate (86)\n      :                 +- Exchange (85)\n      :                    +- HashAggregate (84)\n      :                       +- HashAggregate (83)\n      :                          +- Exchange (82)\n      :                             +- HashAggregate (36)\n      :                                +- Project (35)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (34)\n      +- BroadcastExchange (97)\n         +- HashAggregate (96)\n            +- Exchange (95)\n               +- HashAggregate (94)\n                  +- HashAggregate (93)\n                     +- Exchange (92)\n                        +- HashAggregate (53)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (52)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1792]\n\n(12) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(13) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(14) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(15) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(16) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(17) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(18) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(19) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(20) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(21) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(22) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(23) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(24) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(28) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(29) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(30) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(31) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(32) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(33) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(34) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(35) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(37) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(42) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(43) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(44) AQEShuffleRead\nInput [2]: [course_id#279, count#549L]\nArguments: coalesced\n\n(45) HashAggregate [codegen id : 12]\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(46) HashAggregate [codegen id : 12]\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(47) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]\n\n(48) ShuffleQueryStage\nOutput [2]: [sum#545, count#546L]\nArguments: 11\n\n(49) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(50) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2389]\n\n(51) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(52) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(53) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(54) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(55) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(56) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(57) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(58) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(59) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(60) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(61) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(62) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2254]\n\n(63) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(64) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(65) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(66) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(67) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(68) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(69) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(70) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(71) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(72) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(73) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(74) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(75) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(76) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(77) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(78) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(79) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(80) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(81) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(82) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(83) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(84) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(85) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(86) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(87) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(88) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(89) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(90) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(91) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(92) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(93) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(94) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(95) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(96) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(97) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(98) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(99) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(100) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1792]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2978,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2979,"metricType":"timing"},{"name":"peak memory","accumulatorId":2977,"metricType":"size"},{"name":"number of output rows","accumulatorId":2976,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2981,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2980,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2975,"metricType":"timing"},{"name":"time to build","accumulatorId":2974,"metricType":"timing"},{"name":"time to collect","accumulatorId":2973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2972,"metricType":"sum"},{"name":"data size","accumulatorId":2971,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2941,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3490,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3832,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2389]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 11","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3928,"metricType":"sum"},{"name":"partition data size","accumulatorId":3929,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3930,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3924,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3925,"metricType":"timing"},{"name":"peak memory","accumulatorId":3923,"metricType":"size"},{"name":"number of output rows","accumulatorId":3922,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3927,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3926,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3918,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3917,"metricType":"size"},{"name":"number of output rows","accumulatorId":3916,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3921,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3920,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3904,"metricType":"sum"},{"name":"records read","accumulatorId":3901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3895,"metricType":"sum"},{"name":"data size","accumulatorId":3893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3909,"metricType":"size"},{"name":"number of partitions","accumulatorId":3894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3889,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3890,"metricType":"timing"},{"name":"peak memory","accumulatorId":3888,"metricType":"size"},{"name":"number of output rows","accumulatorId":3887,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3892,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3891,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3886,"metricType":"timing"},{"name":"time to build","accumulatorId":3885,"metricType":"timing"},{"name":"time to collect","accumulatorId":3884,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3883,"metricType":"sum"},{"name":"data size","accumulatorId":3882,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3881,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2254]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3753,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3754,"metricType":"timing"},{"name":"peak memory","accumulatorId":3752,"metricType":"size"},{"name":"number of output rows","accumulatorId":3751,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3756,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3755,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3750,"metricType":"timing"},{"name":"time to build","accumulatorId":3749,"metricType":"timing"},{"name":"time to collect","accumulatorId":3748,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3747,"metricType":"sum"},{"name":"data size","accumulatorId":3746,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3880,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3928,1],[3930,1],[3929,324]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3894,1]]}
{"Event":"SparkListenerJobStart","Job ID":33,"Submission Time":1784293432029,"Stage Infos":[{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"276\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"280\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[62],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":61,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"262\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"258\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[61],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[63,61,62],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"276\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"280\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[62],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432030,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":63,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432034,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":45,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431721,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432053,"Failed":false,"Killed":false,"Accumulables":[{"ID":2634,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2637,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2982,"Name":"data size","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3001,"Name":"shuffle bytes written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3002,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3003,"Name":"shuffle write time","Update":"545120","Value":"545120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3004,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3312,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":3313,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6155058,"Value":6155058,"Internal":true,"Count Failed Values":true},{"ID":3314,"Name":"internal.metrics.executorRunTime","Update":308,"Value":308,"Internal":true,"Count Failed Values":true},{"ID":3315,"Name":"internal.metrics.executorCpuTime","Update":13640468,"Value":13640468,"Internal":true,"Count Failed Values":true},{"ID":3316,"Name":"internal.metrics.resultSize","Update":1879,"Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3340,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3341,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3342,"Name":"internal.metrics.shuffle.write.writeTime","Update":545120,"Value":545120,"Internal":true,"Count Failed Values":true},{"ID":3344,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":6155058,"Executor Run Time":308,"Executor CPU Time":13640468,"Peak Execution Memory":0,"Result Size":1879,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":545120,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"219\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431715,"Completion Time":1784293432056,"Accumulables":[{"ID":2634,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2637,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2640,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2982,"Name":"data size","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3001,"Name":"shuffle bytes written","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3002,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3003,"Name":"shuffle write time","Value":"545120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3004,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3312,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":3313,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6155058,"Internal":true,"Count Failed Values":true},{"ID":3314,"Name":"internal.metrics.executorRunTime","Value":308,"Internal":true,"Count Failed Values":true},{"ID":3315,"Name":"internal.metrics.executorCpuTime","Value":13640468,"Internal":true,"Count Failed Values":true},{"ID":3316,"Name":"internal.metrics.resultSize","Value":1879,"Internal":true,"Count Failed Values":true},{"ID":3340,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3341,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3342,"Name":"internal.metrics.shuffle.write.writeTime","Value":545120,"Internal":true,"Count Failed Values":true},{"ID":3344,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":23,"Completion Time":1784293432057,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":60,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431998,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432061,"Failed":false,"Killed":false,"Accumulables":[{"ID":3555,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3558,"Name":"local bytes read","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3559,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3560,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3838,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3839,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3842,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3845,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":3846,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6880920,"Value":6880920,"Internal":true,"Count Failed Values":true},{"ID":3847,"Name":"internal.metrics.executorRunTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":3848,"Name":"internal.metrics.executorCpuTime","Update":11193436,"Value":11193436,"Internal":true,"Count Failed Values":true},{"ID":3849,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3856,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3857,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3858,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3859,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3860,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3861,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3862,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3863,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3864,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3865,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3866,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3867,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3868,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3869,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3870,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3871,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3872,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":6880920,"Executor Run Time":34,"Executor CPU Time":11193436,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":59,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":101,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"269\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"274\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"270\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[59],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431993,"Completion Time":1784293432063,"Accumulables":[{"ID":3555,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3558,"Name":"local bytes read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3559,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3560,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3838,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3839,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3842,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3845,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":3846,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6880920,"Internal":true,"Count Failed Values":true},{"ID":3847,"Name":"internal.metrics.executorRunTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":3848,"Name":"internal.metrics.executorCpuTime","Value":11193436,"Internal":true,"Count Failed Values":true},{"ID":3849,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3856,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3857,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3858,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3859,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3860,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3861,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3862,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3863,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3864,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3865,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3866,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3867,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3868,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3869,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3870,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3871,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3872,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":32,"Completion Time":1784293432064,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (101)\n+- == Current Plan ==\n   Project (65)\n   +- BroadcastNestedLoopJoin Cross BuildRight (64)\n      :- BroadcastNestedLoopJoin Cross BuildRight (52)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (34)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (21)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (13)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastQueryStage (12)\n      :  :  :  :     +- BroadcastExchange (11)\n      :  :  :  :        +- * HashAggregate (10)\n      :  :  :  :           +- ShuffleQueryStage (9), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :              +- Exchange (8)\n      :  :  :  :                 +- * HashAggregate (7)\n      :  :  :  :                    +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (20), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (19)\n      :  :  :        +- * HashAggregate (18)\n      :  :  :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (16)\n      :  :  :                 +- * HashAggregate (15)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastQueryStage (33)\n      :  :     +- BroadcastExchange (32)\n      :  :        +- * HashAggregate (31)\n      :  :           +- ShuffleQueryStage (30), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (29)\n      :  :                 +- * HashAggregate (28)\n      :  :                    +- * HashAggregate (27)\n      :  :                       +- AQEShuffleRead (26)\n      :  :                          +- ShuffleQueryStage (25), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (24)\n      :  :                                +- * HashAggregate (23)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastExchange (51)\n      :     +- HashAggregate (50)\n      :        +- ShuffleQueryStage (49)\n      :           +- Exchange (48)\n      :              +- * HashAggregate (47)\n      :                 +- * HashAggregate (46)\n      :                    +- AQEShuffleRead (45)\n      :                       +- ShuffleQueryStage (44), Statistics(sizeInBytes=120.0 B, rowCount=5)\n      :                          +- Exchange (43)\n      :                             +- * HashAggregate (42)\n      :                                +- * HashAggregate (41)\n      :                                   +- AQEShuffleRead (40)\n      :                                      +- ShuffleQueryStage (39), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                         +- Exchange (38)\n      :                                            +- * HashAggregate (37)\n      :                                               +- * Project (36)\n      :                                                  +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (63)\n         +- HashAggregate (62)\n            +- ShuffleQueryStage (61)\n               +- Exchange (60)\n                  +- * HashAggregate (59)\n                     +- * HashAggregate (58)\n                        +- AQEShuffleRead (57)\n                           +- ShuffleQueryStage (56), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                              +- Exchange (55)\n                                 +- * HashAggregate (54)\n                                    +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (53)\n+- == Initial Plan ==\n   Project (100)\n   +- BroadcastNestedLoopJoin Cross BuildRight (99)\n      :- BroadcastNestedLoopJoin Cross BuildRight (92)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (82)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (75)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (71)\n      :  :  :  :  :- HashAggregate (67)\n      :  :  :  :  :  +- Exchange (66)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (70)\n      :  :  :  :     +- HashAggregate (69)\n      :  :  :  :        +- Exchange (68)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (74)\n      :  :  :     +- HashAggregate (73)\n      :  :  :        +- Exchange (72)\n      :  :  :           +- HashAggregate (15)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastExchange (81)\n      :  :     +- HashAggregate (80)\n      :  :        +- Exchange (79)\n      :  :           +- HashAggregate (78)\n      :  :              +- HashAggregate (77)\n      :  :                 +- Exchange (76)\n      :  :                    +- HashAggregate (23)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastExchange (91)\n      :     +- HashAggregate (90)\n      :        +- Exchange (89)\n      :           +- HashAggregate (88)\n      :              +- HashAggregate (87)\n      :                 +- Exchange (86)\n      :                    +- HashAggregate (85)\n      :                       +- HashAggregate (84)\n      :                          +- Exchange (83)\n      :                             +- HashAggregate (37)\n      :                                +- Project (36)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (98)\n         +- HashAggregate (97)\n            +- Exchange (96)\n               +- HashAggregate (95)\n                  +- HashAggregate (94)\n                     +- Exchange (93)\n                        +- HashAggregate (54)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (53)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate [codegen id : 13]\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=2436]\n\n(12) BroadcastQueryStage\nOutput [1]: [total_courses#223L]\nArguments: 12\n\n(13) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(14) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(15) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(16) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(17) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(18) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(19) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(20) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(21) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(23) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(24) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(25) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(28) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(29) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(30) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(31) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(32) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(33) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(34) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(35) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(36) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(38) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(40) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(43) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(44) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(45) AQEShuffleRead\nInput [2]: [course_id#279, count#549L]\nArguments: coalesced\n\n(46) HashAggregate [codegen id : 12]\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(47) HashAggregate [codegen id : 12]\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(48) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]\n\n(49) ShuffleQueryStage\nOutput [2]: [sum#545, count#546L]\nArguments: 11\n\n(50) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(51) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2389]\n\n(52) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(53) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(54) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(55) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(56) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(57) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(58) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(59) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(60) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(61) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(62) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(63) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2254]\n\n(64) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(65) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(66) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(67) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(68) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(69) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(70) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(71) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(72) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(73) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(74) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(75) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(76) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(77) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(78) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(79) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(80) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(81) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(82) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(83) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(84) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(85) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(86) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(87) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(88) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(89) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(90) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(91) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(92) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(93) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(94) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(95) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(96) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(97) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(98) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(99) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(100) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(101) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2436]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3979,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3980,"metricType":"timing"},{"name":"peak memory","accumulatorId":3978,"metricType":"size"},{"name":"number of output rows","accumulatorId":3977,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3982,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3981,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3976,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3975,"metricType":"timing"},{"name":"time to build","accumulatorId":3974,"metricType":"timing"},{"name":"time to collect","accumulatorId":3973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3972,"metricType":"sum"},{"name":"data size","accumulatorId":3971,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3970,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3969,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3968,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2389]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 11","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3928,"metricType":"sum"},{"name":"partition data size","accumulatorId":3929,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3930,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3924,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3925,"metricType":"timing"},{"name":"peak memory","accumulatorId":3923,"metricType":"size"},{"name":"number of output rows","accumulatorId":3922,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3927,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3926,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3918,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3917,"metricType":"size"},{"name":"number of output rows","accumulatorId":3916,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3921,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3920,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3904,"metricType":"sum"},{"name":"records read","accumulatorId":3901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3895,"metricType":"sum"},{"name":"data size","accumulatorId":3893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3909,"metricType":"size"},{"name":"number of partitions","accumulatorId":3894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3889,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3890,"metricType":"timing"},{"name":"peak memory","accumulatorId":3888,"metricType":"size"},{"name":"number of output rows","accumulatorId":3887,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3892,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3891,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3886,"metricType":"timing"},{"name":"time to build","accumulatorId":3885,"metricType":"timing"},{"name":"time to collect","accumulatorId":3884,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3883,"metricType":"sum"},{"name":"data size","accumulatorId":3882,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3967,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2254]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3753,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3754,"metricType":"timing"},{"name":"peak memory","accumulatorId":3752,"metricType":"size"},{"name":"number of output rows","accumulatorId":3751,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3756,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3755,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3750,"metricType":"timing"},{"name":"time to build","accumulatorId":3749,"metricType":"timing"},{"name":"time to collect","accumulatorId":3748,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3747,"metricType":"sum"},{"name":"data size","accumulatorId":3746,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3966,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3837,8],[3836,0],[3835,89],[3834,1],[3833,16]]}
{"Event":"SparkListenerTaskEnd","Stage ID":63,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432034,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432096,"Failed":false,"Killed":false,"Accumulables":[{"ID":3675,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3678,"Name":"local bytes read","Update":"312","Value":"312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3679,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3680,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3893,"Name":"data size","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3912,"Name":"shuffle bytes written","Update":"61","Value":"61","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3913,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3914,"Name":"shuffle write time","Update":"321257","Value":"321257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3915,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3916,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3919,"Name":"time in aggregation build","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3922,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3923,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3925,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3926,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3931,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":3932,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9845254,"Value":9845254,"Internal":true,"Count Failed Values":true},{"ID":3933,"Name":"internal.metrics.executorRunTime","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":3934,"Name":"internal.metrics.executorCpuTime","Update":19225114,"Value":19225114,"Internal":true,"Count Failed Values":true},{"ID":3935,"Name":"internal.metrics.resultSize","Update":7318,"Value":7318,"Internal":true,"Count Failed Values":true},{"ID":3940,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3942,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3943,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3944,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3945,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3946,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":312,"Value":312,"Internal":true,"Count Failed Values":true},{"ID":3947,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3948,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":3949,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3950,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3951,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3952,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3953,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3954,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3955,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3956,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3957,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3958,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3959,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":3960,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3961,"Name":"internal.metrics.shuffle.write.writeTime","Update":321257,"Value":321257,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":9845254,"Executor Run Time":36,"Executor CPU Time":19225114,"Peak Execution Memory":4456432,"Result Size":7318,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":312,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":61,"Shuffle Write Time":321257,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"276\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"280\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[62],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432030,"Completion Time":1784293432097,"Accumulables":[{"ID":3675,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3678,"Name":"local bytes read","Value":"312","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3679,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3680,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3893,"Name":"data size","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3912,"Name":"shuffle bytes written","Value":"61","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3913,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3914,"Name":"shuffle write time","Value":"321257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3915,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3916,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3919,"Name":"time in aggregation build","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3922,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3923,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3925,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3926,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3931,"Name":"internal.metrics.executorDeserializeTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":3932,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9845254,"Internal":true,"Count Failed Values":true},{"ID":3933,"Name":"internal.metrics.executorRunTime","Value":36,"Internal":true,"Count Failed Values":true},{"ID":3934,"Name":"internal.metrics.executorCpuTime","Value":19225114,"Internal":true,"Count Failed Values":true},{"ID":3935,"Name":"internal.metrics.resultSize","Value":7318,"Internal":true,"Count Failed Values":true},{"ID":3940,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3942,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3943,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3944,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3945,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3946,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":312,"Internal":true,"Count Failed Values":true},{"ID":3947,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3948,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":3949,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3950,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3951,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3952,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3953,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3954,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3955,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3956,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3957,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3958,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3959,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":61,"Internal":true,"Count Failed Values":true},{"ID":3960,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3961,"Name":"internal.metrics.shuffle.write.writeTime","Value":321257,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":33,"Completion Time":1784293432098,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":34,"Submission Time":1784293432099,"Stage Infos":[{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":78,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"218\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"219\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"295\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[64],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[64,65],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"290\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 7ebe719d-6df3-405d-ad96-fe83fcf94ff1)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"295\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[64],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432101,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"290\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 7ebe719d-6df3-405d-ad96-fe83fcf94ff1)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (102)\n+- == Current Plan ==\n   Project (66)\n   +- BroadcastNestedLoopJoin Cross BuildRight (65)\n      :- BroadcastNestedLoopJoin Cross BuildRight (53)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (34)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (21)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (13)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastQueryStage (12)\n      :  :  :  :     +- BroadcastExchange (11)\n      :  :  :  :        +- * HashAggregate (10)\n      :  :  :  :           +- ShuffleQueryStage (9), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :              +- Exchange (8)\n      :  :  :  :                 +- * HashAggregate (7)\n      :  :  :  :                    +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (20), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (19)\n      :  :  :        +- * HashAggregate (18)\n      :  :  :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (16)\n      :  :  :                 +- * HashAggregate (15)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastQueryStage (33), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :     +- BroadcastExchange (32)\n      :  :        +- * HashAggregate (31)\n      :  :           +- ShuffleQueryStage (30), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (29)\n      :  :                 +- * HashAggregate (28)\n      :  :                    +- * HashAggregate (27)\n      :  :                       +- AQEShuffleRead (26)\n      :  :                          +- ShuffleQueryStage (25), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (24)\n      :  :                                +- * HashAggregate (23)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastQueryStage (52)\n      :     +- BroadcastExchange (51)\n      :        +- * HashAggregate (50)\n      :           +- ShuffleQueryStage (49), Statistics(sizeInBytes=24.0 B, rowCount=1)\n      :              +- Exchange (48)\n      :                 +- * HashAggregate (47)\n      :                    +- * HashAggregate (46)\n      :                       +- AQEShuffleRead (45)\n      :                          +- ShuffleQueryStage (44), Statistics(sizeInBytes=120.0 B, rowCount=5)\n      :                             +- Exchange (43)\n      :                                +- * HashAggregate (42)\n      :                                   +- * HashAggregate (41)\n      :                                      +- AQEShuffleRead (40)\n      :                                         +- ShuffleQueryStage (39), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                            +- Exchange (38)\n      :                                               +- * HashAggregate (37)\n      :                                                  +- * Project (36)\n      :                                                     +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (64)\n         +- HashAggregate (63)\n            +- ShuffleQueryStage (62)\n               +- Exchange (61)\n                  +- * HashAggregate (60)\n                     +- * HashAggregate (59)\n                        +- AQEShuffleRead (58)\n                           +- ShuffleQueryStage (57), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                              +- Exchange (56)\n                                 +- * HashAggregate (55)\n                                    +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n+- == Initial Plan ==\n   Project (101)\n   +- BroadcastNestedLoopJoin Cross BuildRight (100)\n      :- BroadcastNestedLoopJoin Cross BuildRight (93)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (83)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (76)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (72)\n      :  :  :  :  :- HashAggregate (68)\n      :  :  :  :  :  +- Exchange (67)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (71)\n      :  :  :  :     +- HashAggregate (70)\n      :  :  :  :        +- Exchange (69)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (75)\n      :  :  :     +- HashAggregate (74)\n      :  :  :        +- Exchange (73)\n      :  :  :           +- HashAggregate (15)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastExchange (82)\n      :  :     +- HashAggregate (81)\n      :  :        +- Exchange (80)\n      :  :           +- HashAggregate (79)\n      :  :              +- HashAggregate (78)\n      :  :                 +- Exchange (77)\n      :  :                    +- HashAggregate (23)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastExchange (92)\n      :     +- HashAggregate (91)\n      :        +- Exchange (90)\n      :           +- HashAggregate (89)\n      :              +- HashAggregate (88)\n      :                 +- Exchange (87)\n      :                    +- HashAggregate (86)\n      :                       +- HashAggregate (85)\n      :                          +- Exchange (84)\n      :                             +- HashAggregate (37)\n      :                                +- Project (36)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (99)\n         +- HashAggregate (98)\n            +- Exchange (97)\n               +- HashAggregate (96)\n                  +- HashAggregate (95)\n                     +- Exchange (94)\n                        +- HashAggregate (55)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate [codegen id : 13]\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=2436]\n\n(12) BroadcastQueryStage\nOutput [1]: [total_courses#223L]\nArguments: 12\n\n(13) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(14) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(15) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(16) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(17) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(18) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(19) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(20) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(21) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(23) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(24) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(25) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(28) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(29) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(30) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(31) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(32) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(33) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(34) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(35) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(36) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(38) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(40) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(43) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(44) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(45) AQEShuffleRead\nInput [2]: [course_id#279, count#549L]\nArguments: coalesced\n\n(46) HashAggregate [codegen id : 12]\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(47) HashAggregate [codegen id : 12]\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(48) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]\n\n(49) ShuffleQueryStage\nOutput [2]: [sum#545, count#546L]\nArguments: 11\n\n(50) HashAggregate [codegen id : 14]\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(51) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2525]\n\n(52) BroadcastQueryStage\nOutput [1]: [avg_lessons_per_course#252]\nArguments: 13\n\n(53) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(54) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(55) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(56) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(57) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(58) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(59) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(60) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(61) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(62) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(63) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(64) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2254]\n\n(65) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(66) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(67) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(68) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(69) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(70) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(71) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(72) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(73) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(74) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(75) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(76) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(77) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(78) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(79) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(80) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(81) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(82) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(83) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(84) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(85) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(86) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(87) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(88) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(89) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(90) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(91) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(92) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(93) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(94) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(95) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(96) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(97) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(98) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(99) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(100) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(101) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(102) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2436]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3979,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3980,"metricType":"timing"},{"name":"peak memory","accumulatorId":3978,"metricType":"size"},{"name":"number of output rows","accumulatorId":3977,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3982,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3981,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3976,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3975,"metricType":"timing"},{"name":"time to build","accumulatorId":3974,"metricType":"timing"},{"name":"time to collect","accumulatorId":3973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3972,"metricType":"sum"},{"name":"data size","accumulatorId":3971,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3970,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3969,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3968,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 13","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2525]","children":[{"nodeName":"WholeStageCodegen (14)","simpleString":"WholeStageCodegen (14)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 11","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3928,"metricType":"sum"},{"name":"partition data size","accumulatorId":3929,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3930,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3924,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3925,"metricType":"timing"},{"name":"peak memory","accumulatorId":3923,"metricType":"size"},{"name":"number of output rows","accumulatorId":3922,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3927,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3926,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3918,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3917,"metricType":"size"},{"name":"number of output rows","accumulatorId":3916,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3921,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3920,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3904,"metricType":"sum"},{"name":"records read","accumulatorId":3901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3895,"metricType":"sum"},{"name":"data size","accumulatorId":3893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3909,"metricType":"size"},{"name":"number of partitions","accumulatorId":3894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4028,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4029,"metricType":"timing"},{"name":"peak memory","accumulatorId":4027,"metricType":"size"},{"name":"number of output rows","accumulatorId":4026,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4031,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4030,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4025,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4024,"metricType":"timing"},{"name":"time to build","accumulatorId":4023,"metricType":"timing"},{"name":"time to collect","accumulatorId":4022,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4021,"metricType":"sum"},{"name":"data size","accumulatorId":4020,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4019,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2254]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3753,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3754,"metricType":"timing"},{"name":"peak memory","accumulatorId":3752,"metricType":"size"},{"name":"number of output rows","accumulatorId":3751,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3756,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3755,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3750,"metricType":"timing"},{"name":"time to build","accumulatorId":3749,"metricType":"timing"},{"name":"time to collect","accumulatorId":3748,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3747,"metricType":"sum"},{"name":"data size","accumulatorId":3746,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4018,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":57,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293431972,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432099,"Failed":false,"Killed":false,"Accumulables":[{"ID":3257,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3260,"Name":"local bytes read","Update":"891","Value":"891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3261,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3262,"Name":"records read","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3757,"Name":"data size","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3776,"Name":"shuffle bytes written","Update":"62","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3777,"Name":"shuffle records written","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3778,"Name":"shuffle write time","Update":"600254","Value":"600254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3779,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3780,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3783,"Name":"time in aggregation build","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3786,"Name":"number of output rows","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3787,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3789,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3790,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3795,"Name":"internal.metrics.executorDeserializeTime","Update":90,"Value":90,"Internal":true,"Count Failed Values":true},{"ID":3796,"Name":"internal.metrics.executorDeserializeCpuTime","Update":21939203,"Value":21939203,"Internal":true,"Count Failed Values":true},{"ID":3797,"Name":"internal.metrics.executorRunTime","Update":28,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":3798,"Name":"internal.metrics.executorCpuTime","Update":19045454,"Value":19045454,"Internal":true,"Count Failed Values":true},{"ID":3799,"Name":"internal.metrics.resultSize","Update":5358,"Value":5358,"Internal":true,"Count Failed Values":true},{"ID":3804,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3806,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3807,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3808,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3809,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3810,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":891,"Value":891,"Internal":true,"Count Failed Values":true},{"ID":3811,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3812,"Name":"internal.metrics.shuffle.read.recordsRead","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":3813,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3814,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3815,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3816,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3817,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3818,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3819,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3820,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3821,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3822,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3823,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":3824,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3825,"Name":"internal.metrics.shuffle.write.writeTime","Update":600254,"Value":600254,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":90,"Executor Deserialize CPU Time":21939203,"Executor Run Time":28,"Executor CPU Time":19045454,"Peak Execution Memory":4456432,"Result Size":5358,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":891,"Total Records Read":15,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":62,"Shuffle Write Time":600254,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"268\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"264\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293431966,"Completion Time":1784293432110,"Accumulables":[{"ID":3257,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3260,"Name":"local bytes read","Value":"891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3261,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3262,"Name":"records read","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3757,"Name":"data size","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3776,"Name":"shuffle bytes written","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3777,"Name":"shuffle records written","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3778,"Name":"shuffle write time","Value":"600254","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3779,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3780,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3783,"Name":"time in aggregation build","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3786,"Name":"number of output rows","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3787,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3789,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3790,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3795,"Name":"internal.metrics.executorDeserializeTime","Value":90,"Internal":true,"Count Failed Values":true},{"ID":3796,"Name":"internal.metrics.executorDeserializeCpuTime","Value":21939203,"Internal":true,"Count Failed Values":true},{"ID":3797,"Name":"internal.metrics.executorRunTime","Value":28,"Internal":true,"Count Failed Values":true},{"ID":3798,"Name":"internal.metrics.executorCpuTime","Value":19045454,"Internal":true,"Count Failed Values":true},{"ID":3799,"Name":"internal.metrics.resultSize","Value":5358,"Internal":true,"Count Failed Values":true},{"ID":3804,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3806,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3807,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3808,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3809,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3810,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":891,"Internal":true,"Count Failed Values":true},{"ID":3811,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3812,"Name":"internal.metrics.shuffle.read.recordsRead","Value":15,"Internal":true,"Count Failed Values":true},{"ID":3813,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3814,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3815,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3816,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3817,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3818,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3819,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3820,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3821,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3822,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3823,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":62,"Internal":true,"Count Failed Values":true},{"ID":3824,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3825,"Name":"internal.metrics.shuffle.write.writeTime","Value":600254,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":31,"Completion Time":1784293432110,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskStart","Stage ID":65,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432109,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (103)\n+- == Current Plan ==\n   Project (67)\n   +- BroadcastNestedLoopJoin Cross BuildRight (66)\n      :- BroadcastNestedLoopJoin Cross BuildRight (53)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (34)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (21)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (13)\n      :  :  :  :  :- HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastQueryStage (12)\n      :  :  :  :     +- BroadcastExchange (11)\n      :  :  :  :        +- * HashAggregate (10)\n      :  :  :  :           +- ShuffleQueryStage (9), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :              +- Exchange (8)\n      :  :  :  :                 +- * HashAggregate (7)\n      :  :  :  :                    +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (20), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (19)\n      :  :  :        +- * HashAggregate (18)\n      :  :  :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (16)\n      :  :  :                 +- * HashAggregate (15)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastQueryStage (33), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :     +- BroadcastExchange (32)\n      :  :        +- * HashAggregate (31)\n      :  :           +- ShuffleQueryStage (30), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (29)\n      :  :                 +- * HashAggregate (28)\n      :  :                    +- * HashAggregate (27)\n      :  :                       +- AQEShuffleRead (26)\n      :  :                          +- ShuffleQueryStage (25), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (24)\n      :  :                                +- * HashAggregate (23)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastQueryStage (52)\n      :     +- BroadcastExchange (51)\n      :        +- * HashAggregate (50)\n      :           +- ShuffleQueryStage (49), Statistics(sizeInBytes=24.0 B, rowCount=1)\n      :              +- Exchange (48)\n      :                 +- * HashAggregate (47)\n      :                    +- * HashAggregate (46)\n      :                       +- AQEShuffleRead (45)\n      :                          +- ShuffleQueryStage (44), Statistics(sizeInBytes=120.0 B, rowCount=5)\n      :                             +- Exchange (43)\n      :                                +- * HashAggregate (42)\n      :                                   +- * HashAggregate (41)\n      :                                      +- AQEShuffleRead (40)\n      :                                         +- ShuffleQueryStage (39), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                            +- Exchange (38)\n      :                                               +- * HashAggregate (37)\n      :                                                  +- * Project (36)\n      :                                                     +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastQueryStage (65)\n         +- BroadcastExchange (64)\n            +- * HashAggregate (63)\n               +- ShuffleQueryStage (62), Statistics(sizeInBytes=24.0 B, rowCount=1)\n                  +- Exchange (61)\n                     +- * HashAggregate (60)\n                        +- * HashAggregate (59)\n                           +- AQEShuffleRead (58)\n                              +- ShuffleQueryStage (57), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                                 +- Exchange (56)\n                                    +- * HashAggregate (55)\n                                       +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n+- == Initial Plan ==\n   Project (102)\n   +- BroadcastNestedLoopJoin Cross BuildRight (101)\n      :- BroadcastNestedLoopJoin Cross BuildRight (94)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (84)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (77)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (73)\n      :  :  :  :  :- HashAggregate (69)\n      :  :  :  :  :  +- Exchange (68)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (72)\n      :  :  :  :     +- HashAggregate (71)\n      :  :  :  :        +- Exchange (70)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (76)\n      :  :  :     +- HashAggregate (75)\n      :  :  :        +- Exchange (74)\n      :  :  :           +- HashAggregate (15)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastExchange (83)\n      :  :     +- HashAggregate (82)\n      :  :        +- Exchange (81)\n      :  :           +- HashAggregate (80)\n      :  :              +- HashAggregate (79)\n      :  :                 +- Exchange (78)\n      :  :                    +- HashAggregate (23)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastExchange (93)\n      :     +- HashAggregate (92)\n      :        +- Exchange (91)\n      :           +- HashAggregate (90)\n      :              +- HashAggregate (89)\n      :                 +- Exchange (88)\n      :                    +- HashAggregate (87)\n      :                       +- HashAggregate (86)\n      :                          +- Exchange (85)\n      :                             +- HashAggregate (37)\n      :                                +- Project (36)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (100)\n         +- HashAggregate (99)\n            +- Exchange (98)\n               +- HashAggregate (97)\n                  +- HashAggregate (96)\n                     +- Exchange (95)\n                        +- HashAggregate (55)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate [codegen id : 13]\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=2436]\n\n(12) BroadcastQueryStage\nOutput [1]: [total_courses#223L]\nArguments: 12\n\n(13) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(14) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(15) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(16) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(17) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(18) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(19) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(20) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(21) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(23) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(24) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(25) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(28) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(29) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(30) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(31) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(32) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(33) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(34) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(35) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(36) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(38) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(40) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(43) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(44) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(45) AQEShuffleRead\nInput [2]: [course_id#279, count#549L]\nArguments: coalesced\n\n(46) HashAggregate [codegen id : 12]\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(47) HashAggregate [codegen id : 12]\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(48) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]\n\n(49) ShuffleQueryStage\nOutput [2]: [sum#545, count#546L]\nArguments: 11\n\n(50) HashAggregate [codegen id : 14]\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(51) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2525]\n\n(52) BroadcastQueryStage\nOutput [1]: [avg_lessons_per_course#252]\nArguments: 13\n\n(53) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(54) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(55) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(56) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(57) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(58) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(59) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(60) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(61) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(62) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(63) HashAggregate [codegen id : 15]\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(64) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2570]\n\n(65) BroadcastQueryStage\nOutput [1]: [avg_views_per_lesson#265]\nArguments: 14\n\n(66) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(67) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(68) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(69) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(70) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(71) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(72) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(73) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(74) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(75) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(76) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(77) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(78) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(79) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(80) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(81) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(82) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(83) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(84) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(85) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(86) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(87) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(88) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(89) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(90) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(91) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(92) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(93) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(94) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(95) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(96) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(97) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(98) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(99) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(100) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(101) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(102) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(103) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2945,"metricType":"timing"},{"name":"peak memory","accumulatorId":2943,"metricType":"size"},{"name":"number of output rows","accumulatorId":2942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2946,"metricType":"average"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2436]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3979,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3980,"metricType":"timing"},{"name":"peak memory","accumulatorId":3978,"metricType":"size"},{"name":"number of output rows","accumulatorId":3977,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3982,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3981,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3976,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3975,"metricType":"timing"},{"name":"time to build","accumulatorId":3974,"metricType":"timing"},{"name":"time to collect","accumulatorId":3973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3972,"metricType":"sum"},{"name":"data size","accumulatorId":3971,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3970,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3969,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3968,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 13","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2525]","children":[{"nodeName":"WholeStageCodegen (14)","simpleString":"WholeStageCodegen (14)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 11","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3928,"metricType":"sum"},{"name":"partition data size","accumulatorId":3929,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3930,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3924,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3925,"metricType":"timing"},{"name":"peak memory","accumulatorId":3923,"metricType":"size"},{"name":"number of output rows","accumulatorId":3922,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3927,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3926,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3918,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3917,"metricType":"size"},{"name":"number of output rows","accumulatorId":3916,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3921,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3920,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3904,"metricType":"sum"},{"name":"records read","accumulatorId":3901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3895,"metricType":"sum"},{"name":"data size","accumulatorId":3893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3909,"metricType":"size"},{"name":"number of partitions","accumulatorId":3894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4028,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4029,"metricType":"timing"},{"name":"peak memory","accumulatorId":4027,"metricType":"size"},{"name":"number of output rows","accumulatorId":4026,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4031,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4030,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4025,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4024,"metricType":"timing"},{"name":"time to build","accumulatorId":4023,"metricType":"timing"},{"name":"time to collect","accumulatorId":4022,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4021,"metricType":"sum"},{"name":"data size","accumulatorId":4020,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4019,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 14","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2570]","children":[{"nodeName":"WholeStageCodegen (15)","simpleString":"WholeStageCodegen (15)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4041,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4042,"metricType":"timing"},{"name":"peak memory","accumulatorId":4040,"metricType":"size"},{"name":"number of output rows","accumulatorId":4039,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4044,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4043,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4037,"metricType":"timing"},{"name":"time to build","accumulatorId":4036,"metricType":"timing"},{"name":"time to collect","accumulatorId":4035,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4034,"metricType":"sum"},{"name":"data size","accumulatorId":4033,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4032,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":35,"Submission Time":1784293432131,"Stage Infos":[{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":67,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"257\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":93,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"262\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"258\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[93],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[66],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":68,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"275\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"276\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"280\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[67],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"296\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"297\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"301\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[66,67,68,69],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"296\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 24f9e55c-3bf8-4ed0-a981-abd00c4bdbcf)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"296\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"297\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"301\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432132,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"296\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 24f9e55c-3bf8-4ed0-a981-abd00c4bdbcf)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":69,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432139,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":36,"Submission Time":1784293432154,"Stage Infos":[{"Stage ID":70,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"236\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":71,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"263\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"268\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[86],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"264\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"302\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"303\",\"name\":\"WholeStageCodegen (15)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"307\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[70,71,72],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"302\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId d2000fda-c29f-4dc0-94dd-6643423e43b5)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"302\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"303\",\"name\":\"WholeStageCodegen (15)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"307\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432155,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"302\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId d2000fda-c29f-4dc0-94dd-6643423e43b5)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":72,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432162,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":65,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432109,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432188,"Failed":false,"Killed":false,"Accumulables":[{"ID":2985,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2988,"Name":"local bytes read","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2989,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2990,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3976,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3977,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3980,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3983,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":3984,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7684226,"Value":7684226,"Internal":true,"Count Failed Values":true},{"ID":3985,"Name":"internal.metrics.executorRunTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":3986,"Name":"internal.metrics.executorCpuTime","Update":13029586,"Value":13029586,"Internal":true,"Count Failed Values":true},{"ID":3987,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3994,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3995,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3996,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3997,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3998,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":3999,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4000,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4001,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4002,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4003,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4004,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4005,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4006,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4007,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4008,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4009,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4010,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":7684226,"Executor Run Time":33,"Executor CPU Time":13029586,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":59,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":105,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"295\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (13)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[64],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432101,"Completion Time":1784293432192,"Accumulables":[{"ID":2985,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2988,"Name":"local bytes read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2989,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2990,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3976,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3977,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3980,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3983,"Name":"internal.metrics.executorDeserializeTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":3984,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7684226,"Internal":true,"Count Failed Values":true},{"ID":3985,"Name":"internal.metrics.executorRunTime","Value":33,"Internal":true,"Count Failed Values":true},{"ID":3986,"Name":"internal.metrics.executorCpuTime","Value":13029586,"Internal":true,"Count Failed Values":true},{"ID":3987,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":3994,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3995,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3996,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3997,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3998,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59,"Internal":true,"Count Failed Values":true},{"ID":3999,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4000,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4001,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4002,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4003,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4004,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4005,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4006,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4007,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4008,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4009,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4010,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":34,"Completion Time":1784293432192,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[3975,2],[3974,0],[3973,116],[3972,1],[3971,16]]}
{"Event":"SparkListenerTaskEnd","Stage ID":69,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432139,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432202,"Failed":false,"Killed":false,"Accumulables":[{"ID":3896,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3899,"Name":"local bytes read","Update":"61","Value":"61","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3900,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3901,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4025,"Name":"duration","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4026,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4029,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4045,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":4046,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5571920,"Value":5571920,"Internal":true,"Count Failed Values":true},{"ID":4047,"Name":"internal.metrics.executorRunTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":4048,"Name":"internal.metrics.executorCpuTime","Update":13580844,"Value":13580844,"Internal":true,"Count Failed Values":true},{"ID":4049,"Name":"internal.metrics.resultSize","Update":3995,"Value":3995,"Internal":true,"Count Failed Values":true},{"ID":4056,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4057,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4058,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4059,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4060,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":61,"Value":61,"Internal":true,"Count Failed Values":true},{"ID":4061,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4062,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4063,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4064,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4065,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4066,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4067,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4068,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4069,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4070,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4071,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4072,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":5571920,"Executor Run Time":22,"Executor CPU Time":13580844,"Peak Execution Memory":0,"Result Size":3995,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":61,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"296\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"297\",\"name\":\"WholeStageCodegen (14)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"301\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[68],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432132,"Completion Time":1784293432205,"Accumulables":[{"ID":3896,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3899,"Name":"local bytes read","Value":"61","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3900,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3901,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4025,"Name":"duration","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4026,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4029,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4045,"Name":"internal.metrics.executorDeserializeTime","Value":25,"Internal":true,"Count Failed Values":true},{"ID":4046,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5571920,"Internal":true,"Count Failed Values":true},{"ID":4047,"Name":"internal.metrics.executorRunTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":4048,"Name":"internal.metrics.executorCpuTime","Value":13580844,"Internal":true,"Count Failed Values":true},{"ID":4049,"Name":"internal.metrics.resultSize","Value":3995,"Internal":true,"Count Failed Values":true},{"ID":4056,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4057,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4058,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4059,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4060,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":61,"Internal":true,"Count Failed Values":true},{"ID":4061,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4062,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4063,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4064,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4065,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4066,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4067,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4068,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4069,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4070,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4071,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4072,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":35,"Completion Time":1784293432206,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[4024,2],[4023,0],[4022,97],[4021,1],[4020,16]]}
{"Event":"SparkListenerTaskEnd","Stage ID":72,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432162,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432224,"Failed":false,"Killed":false,"Accumulables":[{"ID":3760,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3763,"Name":"local bytes read","Update":"62","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3764,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3765,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4038,"Name":"duration","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4039,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4042,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4080,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":4081,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5101238,"Value":5101238,"Internal":true,"Count Failed Values":true},{"ID":4082,"Name":"internal.metrics.executorRunTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":4083,"Name":"internal.metrics.executorCpuTime","Update":15356479,"Value":15356479,"Internal":true,"Count Failed Values":true},{"ID":4084,"Name":"internal.metrics.resultSize","Update":4038,"Value":4038,"Internal":true,"Count Failed Values":true},{"ID":4086,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4091,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4092,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4093,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4094,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4095,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":4096,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4097,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4098,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4099,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4100,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4101,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4102,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4103,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4104,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4105,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4106,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4107,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":5101238,"Executor Run Time":30,"Executor CPU Time":15356479,"Peak Execution Memory":0,"Result Size":4038,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":62,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"302\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"303\",\"name\":\"WholeStageCodegen (15)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"307\",\"name\":\"Exchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[98],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432155,"Completion Time":1784293432225,"Accumulables":[{"ID":3760,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3763,"Name":"local bytes read","Value":"62","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3764,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3765,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4038,"Name":"duration","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4039,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4042,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4080,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":4081,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5101238,"Internal":true,"Count Failed Values":true},{"ID":4082,"Name":"internal.metrics.executorRunTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":4083,"Name":"internal.metrics.executorCpuTime","Value":15356479,"Internal":true,"Count Failed Values":true},{"ID":4084,"Name":"internal.metrics.resultSize","Value":4038,"Internal":true,"Count Failed Values":true},{"ID":4086,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4091,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4092,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4093,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4094,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4095,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":62,"Internal":true,"Count Failed Values":true},{"ID":4096,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4097,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4098,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4099,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4100,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4101,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4102,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4103,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4104,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4105,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4106,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4107,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":36,"Completion Time":1784293432225,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[4037,2],[4036,0],[4035,101],[4034,1],[4033,16]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (103)\n+- == Final Plan ==\n   * Project (67)\n   +- * BroadcastNestedLoopJoin Cross BuildRight (66)\n      :- * BroadcastNestedLoopJoin Cross BuildRight (53)\n      :  :- * BroadcastNestedLoopJoin Cross BuildRight (34)\n      :  :  :- * BroadcastNestedLoopJoin Cross BuildRight (21)\n      :  :  :  :- * BroadcastNestedLoopJoin Cross BuildRight (13)\n      :  :  :  :  :- * HashAggregate (5)\n      :  :  :  :  :  +- ShuffleQueryStage (4), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :  :     +- Exchange (3)\n      :  :  :  :  :        +- * HashAggregate (2)\n      :  :  :  :  :           +- * Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastQueryStage (12), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :     +- BroadcastExchange (11)\n      :  :  :  :        +- * HashAggregate (10)\n      :  :  :  :           +- ShuffleQueryStage (9), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :  :              +- Exchange (8)\n      :  :  :  :                 +- * HashAggregate (7)\n      :  :  :  :                    +- * Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastQueryStage (20), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :     +- BroadcastExchange (19)\n      :  :  :        +- * HashAggregate (18)\n      :  :  :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :  :              +- Exchange (16)\n      :  :  :                 +- * HashAggregate (15)\n      :  :  :                    +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastQueryStage (33), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :     +- BroadcastExchange (32)\n      :  :        +- * HashAggregate (31)\n      :  :           +- ShuffleQueryStage (30), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :  :              +- Exchange (29)\n      :  :                 +- * HashAggregate (28)\n      :  :                    +- * HashAggregate (27)\n      :  :                       +- AQEShuffleRead (26)\n      :  :                          +- ShuffleQueryStage (25), Statistics(sizeInBytes=176.0 B, rowCount=11)\n      :  :                             +- Exchange (24)\n      :  :                                +- * HashAggregate (23)\n      :  :                                   +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastQueryStage (52), Statistics(sizeInBytes=16.0 B, rowCount=1)\n      :     +- BroadcastExchange (51)\n      :        +- * HashAggregate (50)\n      :           +- ShuffleQueryStage (49), Statistics(sizeInBytes=24.0 B, rowCount=1)\n      :              +- Exchange (48)\n      :                 +- * HashAggregate (47)\n      :                    +- * HashAggregate (46)\n      :                       +- AQEShuffleRead (45)\n      :                          +- ShuffleQueryStage (44), Statistics(sizeInBytes=120.0 B, rowCount=5)\n      :                             +- Exchange (43)\n      :                                +- * HashAggregate (42)\n      :                                   +- * HashAggregate (41)\n      :                                      +- AQEShuffleRead (40)\n      :                                         +- ShuffleQueryStage (39), Statistics(sizeInBytes=432.0 B, rowCount=18)\n      :                                            +- Exchange (38)\n      :                                               +- * HashAggregate (37)\n      :                                                  +- * Project (36)\n      :                                                     +- * Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastQueryStage (65), Statistics(sizeInBytes=16.0 B, rowCount=1)\n         +- BroadcastExchange (64)\n            +- * HashAggregate (63)\n               +- ShuffleQueryStage (62), Statistics(sizeInBytes=24.0 B, rowCount=1)\n                  +- Exchange (61)\n                     +- * HashAggregate (60)\n                        +- * HashAggregate (59)\n                           +- AQEShuffleRead (58)\n                              +- ShuffleQueryStage (57), Statistics(sizeInBytes=360.0 B, rowCount=15)\n                                 +- Exchange (56)\n                                    +- * HashAggregate (55)\n                                       +- * Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n+- == Initial Plan ==\n   Project (102)\n   +- BroadcastNestedLoopJoin Cross BuildRight (101)\n      :- BroadcastNestedLoopJoin Cross BuildRight (94)\n      :  :- BroadcastNestedLoopJoin Cross BuildRight (84)\n      :  :  :- BroadcastNestedLoopJoin Cross BuildRight (77)\n      :  :  :  :- BroadcastNestedLoopJoin Cross BuildRight (73)\n      :  :  :  :  :- HashAggregate (69)\n      :  :  :  :  :  +- Exchange (68)\n      :  :  :  :  :     +- HashAggregate (2)\n      :  :  :  :  :        +- Scan JDBCRelation(raw.users) [numPartitions=1]  (1)\n      :  :  :  :  +- BroadcastExchange (72)\n      :  :  :  :     +- HashAggregate (71)\n      :  :  :  :        +- Exchange (70)\n      :  :  :  :           +- HashAggregate (7)\n      :  :  :  :              +- Scan JDBCRelation(raw.courses) [numPartitions=1]  (6)\n      :  :  :  +- BroadcastExchange (76)\n      :  :  :     +- HashAggregate (75)\n      :  :  :        +- Exchange (74)\n      :  :  :           +- HashAggregate (15)\n      :  :  :              +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (14)\n      :  :  +- BroadcastExchange (83)\n      :  :     +- HashAggregate (82)\n      :  :        +- Exchange (81)\n      :  :           +- HashAggregate (80)\n      :  :              +- HashAggregate (79)\n      :  :                 +- Exchange (78)\n      :  :                    +- HashAggregate (23)\n      :  :                       +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (22)\n      :  +- BroadcastExchange (93)\n      :     +- HashAggregate (92)\n      :        +- Exchange (91)\n      :           +- HashAggregate (90)\n      :              +- HashAggregate (89)\n      :                 +- Exchange (88)\n      :                    +- HashAggregate (87)\n      :                       +- HashAggregate (86)\n      :                          +- Exchange (85)\n      :                             +- HashAggregate (37)\n      :                                +- Project (36)\n      :                                   +- Scan JDBCRelation(raw.lessons) [numPartitions=1]  (35)\n      +- BroadcastExchange (100)\n         +- HashAggregate (99)\n            +- Exchange (98)\n               +- HashAggregate (97)\n                  +- HashAggregate (96)\n                     +- Exchange (95)\n                        +- HashAggregate (55)\n                           +- Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  (54)\n\n\n(1) Scan JDBCRelation(raw.users) [numPartitions=1]  [codegen id : 1]\nOutput: []\nReadSchema: struct<>\n\n(2) HashAggregate [codegen id : 1]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#533L]\nResults [1]: [count#534L]\n\n(3) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]\n\n(4) ShuffleQueryStage\nOutput [1]: [count#534L]\nArguments: 0\n\n(5) HashAggregate [codegen id : 16]\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(6) Scan JDBCRelation(raw.courses) [numPartitions=1]  [codegen id : 2]\nOutput: []\nReadSchema: struct<>\n\n(7) HashAggregate [codegen id : 2]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#535L]\nResults [1]: [count#536L]\n\n(8) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]\n\n(9) ShuffleQueryStage\nOutput [1]: [count#536L]\nArguments: 1\n\n(10) HashAggregate [codegen id : 13]\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(11) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=2436]\n\n(12) BroadcastQueryStage\nOutput [1]: [total_courses#223L]\nArguments: 12\n\n(13) BroadcastNestedLoopJoin [codegen id : 16]\nJoin type: Cross\nJoin condition: None\n\n(14) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 3]\nOutput: []\nReadSchema: struct<>\n\n(15) HashAggregate [codegen id : 3]\nInput: []\nKeys: []\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#537L]\nResults [1]: [count#538L]\n\n(16) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]\n\n(17) ShuffleQueryStage\nOutput [1]: [count#538L]\nArguments: 2\n\n(18) HashAggregate [codegen id : 7]\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(19) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1963]\n\n(20) BroadcastQueryStage\nOutput [1]: [total_lessons#230L]\nArguments: 6\n\n(21) BroadcastNestedLoopJoin [codegen id : 16]\nJoin type: Cross\nJoin condition: None\n\n(22) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [1]: [user_id#51]\nReadSchema: struct<user_id:int>\n\n(23) HashAggregate [codegen id : 4]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(24) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]\n\n(25) ShuffleQueryStage\nOutput [1]: [user_id#51]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [1]: [user_id#51]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(28) HashAggregate [codegen id : 8]\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(29) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]\n\n(30) ShuffleQueryStage\nOutput [1]: [count#541L]\nArguments: 7\n\n(31) HashAggregate [codegen id : 11]\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(32) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=2309]\n\n(33) BroadcastQueryStage\nOutput [1]: [users_with_views#236L]\nArguments: 10\n\n(34) BroadcastNestedLoopJoin [codegen id : 16]\nJoin type: Cross\nJoin condition: None\n\n(35) Scan JDBCRelation(raw.lessons) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [id#278, course_id#279]\nReadSchema: struct<id:int,course_id:int>\n\n(36) Project [codegen id : 5]\nOutput [2]: [id#278 AS lesson_id#37, course_id#279]\nInput [2]: [id#278, course_id#279]\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#37, course_id#279]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(38) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#279, lesson_id#37]\nArguments: 4\n\n(40) AQEShuffleRead\nInput [2]: [course_id#279, lesson_id#37]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(43) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]\n\n(44) ShuffleQueryStage\nOutput [2]: [course_id#279, count#549L]\nArguments: 8\n\n(45) AQEShuffleRead\nInput [2]: [course_id#279, count#549L]\nArguments: coalesced\n\n(46) HashAggregate [codegen id : 12]\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(47) HashAggregate [codegen id : 12]\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(48) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]\n\n(49) ShuffleQueryStage\nOutput [2]: [sum#545, count#546L]\nArguments: 11\n\n(50) HashAggregate [codegen id : 14]\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(51) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=2525]\n\n(52) BroadcastQueryStage\nOutput [1]: [avg_lessons_per_course#252]\nArguments: 13\n\n(53) BroadcastNestedLoopJoin [codegen id : 16]\nJoin type: Cross\nJoin condition: None\n\n(54) Scan JDBCRelation(raw.lesson_views) [numPartitions=1]  [codegen id : 6]\nOutput [1]: [lesson_id#289]\nReadSchema: struct<lesson_id:int>\n\n(55) HashAggregate [codegen id : 6]\nInput [1]: [lesson_id#289]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [partial_count(1)]\nAggregate Attributes [1]: [count#555L]\nResults [2]: [lesson_id#289, count#556L]\n\n(56) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]\n\n(57) ShuffleQueryStage\nOutput [2]: [lesson_id#289, count#556L]\nArguments: 5\n\n(58) AQEShuffleRead\nInput [2]: [lesson_id#289, count#556L]\nArguments: coalesced\n\n(59) HashAggregate [codegen id : 10]\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(60) HashAggregate [codegen id : 10]\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(61) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]\n\n(62) ShuffleQueryStage\nOutput [2]: [sum#553, count#554L]\nArguments: 9\n\n(63) HashAggregate [codegen id : 15]\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(64) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=2570]\n\n(65) BroadcastQueryStage\nOutput [1]: [avg_views_per_lesson#265]\nArguments: 14\n\n(66) BroadcastNestedLoopJoin [codegen id : 16]\nJoin type: Cross\nJoin condition: None\n\n(67) Project [codegen id : 16]\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(68) Exchange\nInput [1]: [count#534L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1725]\n\n(69) HashAggregate\nInput [1]: [count#534L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#214L]\nResults [1]: [count(1)#214L AS total_users#215L]\n\n(70) Exchange\nInput [1]: [count#536L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1727]\n\n(71) HashAggregate\nInput [1]: [count#536L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#222L]\nResults [1]: [count(1)#222L AS total_courses#223L]\n\n(72) BroadcastExchange\nInput [1]: [total_courses#223L]\nArguments: IdentityBroadcastMode, [plan_id=1730]\n\n(73) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(74) Exchange\nInput [1]: [count#538L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1732]\n\n(75) HashAggregate\nInput [1]: [count#538L]\nKeys: []\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#229L]\nResults [1]: [count(1)#229L AS total_lessons#230L]\n\n(76) BroadcastExchange\nInput [1]: [total_lessons#230L]\nArguments: IdentityBroadcastMode, [plan_id=1735]\n\n(77) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(78) Exchange\nInput [1]: [user_id#51]\nArguments: hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1737]\n\n(79) HashAggregate\nInput [1]: [user_id#51]\nKeys [1]: [user_id#51]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#51]\n\n(80) HashAggregate\nInput [1]: [user_id#51]\nKeys: []\nFunctions [1]: [partial_count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count#541L]\n\n(81) Exchange\nInput [1]: [count#541L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1741]\n\n(82) HashAggregate\nInput [1]: [count#541L]\nKeys: []\nFunctions [1]: [count(distinct user_id#51)]\nAggregate Attributes [1]: [count(user_id#51)#241L]\nResults [1]: [count(user_id#51)#241L AS users_with_views#236L]\n\n(83) BroadcastExchange\nInput [1]: [users_with_views#236L]\nArguments: IdentityBroadcastMode, [plan_id=1744]\n\n(84) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(85) Exchange\nInput [2]: [course_id#279, lesson_id#37]\nArguments: hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1746]\n\n(86) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [2]: [course_id#279, lesson_id#37]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [course_id#279, lesson_id#37]\n\n(87) HashAggregate\nInput [2]: [course_id#279, lesson_id#37]\nKeys [1]: [course_id#279]\nFunctions [1]: [partial_count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [2]: [course_id#279, count#549L]\n\n(88) Exchange\nInput [2]: [course_id#279, count#549L]\nArguments: hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=1750]\n\n(89) HashAggregate\nInput [2]: [course_id#279, count#549L]\nKeys [1]: [course_id#279]\nFunctions [1]: [count(distinct lesson_id#37)]\nAggregate Attributes [1]: [count(lesson_id#37)#248L]\nResults [1]: [count(lesson_id#37)#248L AS lessons_count#247L]\n\n(90) HashAggregate\nInput [1]: [lessons_count#247L]\nKeys: []\nFunctions [1]: [partial_avg(lessons_count#247L)]\nAggregate Attributes [2]: [sum#543, count#544L]\nResults [2]: [sum#545, count#546L]\n\n(91) Exchange\nInput [2]: [sum#545, count#546L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1754]\n\n(92) HashAggregate\nInput [2]: [sum#545, count#546L]\nKeys: []\nFunctions [1]: [avg(lessons_count#247L)]\nAggregate Attributes [1]: [avg(lessons_count#247L)#251]\nResults [1]: [round(avg(lessons_count#247L)#251, 2) AS avg_lessons_per_course#252]\n\n(93) BroadcastExchange\nInput [1]: [avg_lessons_per_course#252]\nArguments: IdentityBroadcastMode, [plan_id=1757]\n\n(94) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(95) Exchange\nInput [2]: [lesson_id#289, count#556L]\nArguments: hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1759]\n\n(96) HashAggregate\nInput [2]: [lesson_id#289, count#556L]\nKeys [1]: [lesson_id#289]\nFunctions [1]: [count(1)]\nAggregate Attributes [1]: [count(1)#260L]\nResults [1]: [count(1)#260L AS total_views#261L]\n\n(97) HashAggregate\nInput [1]: [total_views#261L]\nKeys: []\nFunctions [1]: [partial_avg(total_views#261L)]\nAggregate Attributes [2]: [sum#551, count#552L]\nResults [2]: [sum#553, count#554L]\n\n(98) Exchange\nInput [2]: [sum#553, count#554L]\nArguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1763]\n\n(99) HashAggregate\nInput [2]: [sum#553, count#554L]\nKeys: []\nFunctions [1]: [avg(total_views#261L)]\nAggregate Attributes [1]: [avg(total_views#261L)#264]\nResults [1]: [round(avg(total_views#261L)#264, 2) AS avg_views_per_lesson#265]\n\n(100) BroadcastExchange\nInput [1]: [avg_views_per_lesson#265]\nArguments: IdentityBroadcastMode, [plan_id=1766]\n\n(101) BroadcastNestedLoopJoin\nJoin type: Cross\nJoin condition: None\n\n(102) Project\nOutput [6]: [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]\nInput [6]: [total_users#215L, total_courses#223L, total_lessons#230L, users_with_views#236L, avg_lessons_per_course#252, avg_views_per_lesson#265]\n\n(103) AdaptiveSparkPlan\nOutput [6]: [toprettystring(total_users)#521, toprettystring(total_courses)#522, toprettystring(total_lessons)#523, toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson)#526]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"WholeStageCodegen (16)","simpleString":"WholeStageCodegen (16)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(total_users#215L, Some(Europe/Samara)) AS toprettystring(total_users)#521, toprettystring(total_courses#223L, Some(Europe/Samara)) AS toprettystring(total_courses)#522, toprettystring(total_lessons#230L, Some(Europe/Samara)) AS toprettystring(total_lessons)#523, toprettystring(users_with_views#236L, Some(Europe/Samara)) AS toprettystring(users_with_views)#524, toprettystring(avg_lessons_per_course#252, Some(Europe/Samara)) AS toprettystring(avg_lessons_per_course)#525, toprettystring(avg_views_per_lesson#265, Some(Europe/Samara)) AS toprettystring(avg_views_per_lesson)#526]","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"BroadcastNestedLoopJoin","simpleString":"BroadcastNestedLoopJoin BuildRight, Cross","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1774]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.users) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.users) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2600,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2596,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2597,"metricType":"timing"},{"name":"peak memory","accumulatorId":2595,"metricType":"size"},{"name":"number of output rows","accumulatorId":2594,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2599,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2598,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2970,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2959,"metricType":"sum"},{"name":"records read","accumulatorId":2956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2950,"metricType":"sum"},{"name":"data size","accumulatorId":2948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2964,"metricType":"size"},{"name":"number of partitions","accumulatorId":2949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2967,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4123,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4124,"metricType":"timing"},{"name":"peak memory","accumulatorId":4122,"metricType":"size"},{"name":"number of output rows","accumulatorId":4121,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4126,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4125,"metricType":"average"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 12","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2436]","children":[{"nodeName":"WholeStageCodegen (13)","simpleString":"WholeStageCodegen (13)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1787]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.courses) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.courses) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2640,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2636,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2637,"metricType":"timing"},{"name":"peak memory","accumulatorId":2635,"metricType":"size"},{"name":"number of output rows","accumulatorId":2634,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2639,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2638,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3004,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3002,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2996,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3003,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2997,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2994,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2991,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3000,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2993,"metricType":"sum"},{"name":"records read","accumulatorId":2990,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2988,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2989,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2986,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2992,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2985,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2995,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2984,"metricType":"sum"},{"name":"data size","accumulatorId":2982,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2998,"metricType":"size"},{"name":"number of partitions","accumulatorId":2983,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2999,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2987,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3001,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3979,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3980,"metricType":"timing"},{"name":"peak memory","accumulatorId":3978,"metricType":"size"},{"name":"number of output rows","accumulatorId":3977,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3982,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3981,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3976,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3975,"metricType":"timing"},{"name":"time to build","accumulatorId":3974,"metricType":"timing"},{"name":"time to collect","accumulatorId":3973,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3972,"metricType":"sum"},{"name":"data size","accumulatorId":3971,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4120,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=1963]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1802]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [] PushedFilters: [], ReadSchema: struct<>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2680,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2676,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2677,"metricType":"timing"},{"name":"peak memory","accumulatorId":2675,"metricType":"size"},{"name":"number of output rows","accumulatorId":2674,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2679,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2678,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3036,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3030,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3037,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3031,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3028,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3025,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3034,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3027,"metricType":"sum"},{"name":"records read","accumulatorId":3024,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3022,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3023,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3020,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3026,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3019,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3029,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3018,"metricType":"sum"},{"name":"data size","accumulatorId":3016,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3032,"metricType":"size"},{"name":"number of partitions","accumulatorId":3017,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3033,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3021,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3035,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3499,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3500,"metricType":"timing"},{"name":"peak memory","accumulatorId":3498,"metricType":"size"},{"name":"number of output rows","accumulatorId":3497,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3502,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3501,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3496,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3495,"metricType":"timing"},{"name":"time to build","accumulatorId":3494,"metricType":"timing"},{"name":"time to collect","accumulatorId":3493,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3492,"metricType":"sum"},{"name":"data size","accumulatorId":3491,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4119,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 10","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2309]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[count(distinct user_id#51)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2069]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_count(distinct user_id#51)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#51, 200), ENSURE_REQUIREMENTS, [plan_id=1823]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#51], functions=[])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [user_id#51] PushedFilters: [], ReadSchema: struct<user_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2754,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2750,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2751,"metricType":"timing"},{"name":"peak memory","accumulatorId":2749,"metricType":"size"},{"name":"number of output rows","accumulatorId":2748,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2753,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2752,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3106,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3104,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3098,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3105,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3099,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3096,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3093,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3102,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3095,"metricType":"sum"},{"name":"records read","accumulatorId":3092,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3090,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3091,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3088,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3094,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3087,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3097,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3086,"metricType":"sum"},{"name":"data size","accumulatorId":3084,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3100,"metricType":"size"},{"name":"number of partitions","accumulatorId":3085,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3101,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3089,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3103,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3587,"metricType":"sum"},{"name":"partition data size","accumulatorId":3588,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3589,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3583,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3584,"metricType":"timing"},{"name":"peak memory","accumulatorId":3582,"metricType":"size"},{"name":"number of output rows","accumulatorId":3581,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3586,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3585,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3577,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3578,"metricType":"timing"},{"name":"peak memory","accumulatorId":3576,"metricType":"size"},{"name":"number of output rows","accumulatorId":3575,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3580,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3579,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3574,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3572,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3566,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3573,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3567,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3564,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3561,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3570,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3563,"metricType":"sum"},{"name":"records read","accumulatorId":3560,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3558,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3559,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3556,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3562,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3555,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3565,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3554,"metricType":"sum"},{"name":"data size","accumulatorId":3552,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3568,"metricType":"size"},{"name":"number of partitions","accumulatorId":3553,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3569,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3557,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3571,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3841,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3842,"metricType":"timing"},{"name":"peak memory","accumulatorId":3840,"metricType":"size"},{"name":"number of output rows","accumulatorId":3839,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3844,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3843,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3838,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3837,"metricType":"timing"},{"name":"time to build","accumulatorId":3836,"metricType":"timing"},{"name":"time to collect","accumulatorId":3835,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3834,"metricType":"sum"},{"name":"data size","accumulatorId":3833,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4118,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 13","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2525]","children":[{"nodeName":"WholeStageCodegen (14)","simpleString":"WholeStageCodegen (14)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(lessons_count#247L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 11","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2378]","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(lessons_count#247L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[count(distinct lesson_id#37)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, 200), ENSURE_REQUIREMENTS, [plan_id=2152]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279], functions=[partial_count(distinct lesson_id#37)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#279, lesson_id#37, 200), ENSURE_REQUIREMENTS, [plan_id=1856]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#279, lesson_id#37], functions=[])","children":[{"nodeName":"Project","simpleString":"Project [id#278 AS lesson_id#37, course_id#279]","children":[{"nodeName":"Scan JDBCRelation(raw.lessons) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lessons) [numPartitions=1] [id#278,course_id#279] PushedFilters: [], ReadSchema: struct<id:int,course_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2862,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2858,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2859,"metricType":"timing"},{"name":"peak memory","accumulatorId":2857,"metricType":"size"},{"name":"number of output rows","accumulatorId":2856,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2861,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2860,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3208,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3206,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3200,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3207,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3201,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3198,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3195,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3204,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3197,"metricType":"sum"},{"name":"records read","accumulatorId":3194,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3192,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3193,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3190,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3196,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3189,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3199,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3188,"metricType":"sum"},{"name":"data size","accumulatorId":3186,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3202,"metricType":"size"},{"name":"number of partitions","accumulatorId":3187,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3203,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3191,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3205,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3707,"metricType":"sum"},{"name":"partition data size","accumulatorId":3708,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3709,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3703,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3704,"metricType":"timing"},{"name":"peak memory","accumulatorId":3702,"metricType":"size"},{"name":"number of output rows","accumulatorId":3701,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3706,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3705,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3697,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3698,"metricType":"timing"},{"name":"peak memory","accumulatorId":3696,"metricType":"size"},{"name":"number of output rows","accumulatorId":3695,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3700,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3699,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3694,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3692,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3686,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3693,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3687,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3684,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3681,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3690,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3683,"metricType":"sum"},{"name":"records read","accumulatorId":3680,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3678,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3679,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3676,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3682,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3675,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3685,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3674,"metricType":"sum"},{"name":"data size","accumulatorId":3672,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3688,"metricType":"size"},{"name":"number of partitions","accumulatorId":3673,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3689,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3677,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3691,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3928,"metricType":"sum"},{"name":"partition data size","accumulatorId":3929,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3930,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3924,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3925,"metricType":"timing"},{"name":"peak memory","accumulatorId":3923,"metricType":"size"},{"name":"number of output rows","accumulatorId":3922,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3927,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3926,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3918,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3917,"metricType":"size"},{"name":"number of output rows","accumulatorId":3916,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3921,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3920,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3904,"metricType":"sum"},{"name":"records read","accumulatorId":3901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3895,"metricType":"sum"},{"name":"data size","accumulatorId":3893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3909,"metricType":"size"},{"name":"number of partitions","accumulatorId":3894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4028,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4029,"metricType":"timing"},{"name":"peak memory","accumulatorId":4027,"metricType":"size"},{"name":"number of output rows","accumulatorId":4026,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4031,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4030,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4025,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4024,"metricType":"timing"},{"name":"time to build","accumulatorId":4023,"metricType":"timing"},{"name":"time to collect","accumulatorId":4022,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4021,"metricType":"sum"},{"name":"data size","accumulatorId":4020,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4117,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 14","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange IdentityBroadcastMode, [plan_id=2570]","children":[{"nodeName":"WholeStageCodegen (15)","simpleString":"WholeStageCodegen (15)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[avg(total_views#261L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 9","children":[{"nodeName":"Exchange","simpleString":"Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2243]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[], functions=[partial_avg(total_views#261L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[count(1)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 5","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#289, 200), ENSURE_REQUIREMENTS, [plan_id=1884]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#289], functions=[partial_count(1)])","children":[{"nodeName":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(raw.lesson_views) [numPartitions=1] [lesson_id#289] PushedFilters: [], ReadSchema: struct<lesson_id:int>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2936,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2932,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2933,"metricType":"timing"},{"name":"peak memory","accumulatorId":2931,"metricType":"size"},{"name":"number of output rows","accumulatorId":2930,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2935,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2934,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3265,"metricType":"sum"},{"name":"records read","accumulatorId":3262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3256,"metricType":"sum"},{"name":"data size","accumulatorId":3254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3270,"metricType":"size"},{"name":"number of partitions","accumulatorId":3255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3792,"metricType":"sum"},{"name":"partition data size","accumulatorId":3793,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3794,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3788,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3789,"metricType":"timing"},{"name":"peak memory","accumulatorId":3787,"metricType":"size"},{"name":"number of output rows","accumulatorId":3786,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3791,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3790,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3777,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3771,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3778,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3772,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3769,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3766,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3775,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3768,"metricType":"sum"},{"name":"records read","accumulatorId":3765,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3763,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3764,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3761,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3767,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3760,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3770,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3759,"metricType":"sum"},{"name":"data size","accumulatorId":3757,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3773,"metricType":"size"},{"name":"number of partitions","accumulatorId":3758,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3774,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3762,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3776,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4041,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4042,"metricType":"timing"},{"name":"peak memory","accumulatorId":4040,"metricType":"size"},{"name":"number of output rows","accumulatorId":4039,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4044,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4043,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4038,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4037,"metricType":"timing"},{"name":"time to build","accumulatorId":4036,"metricType":"timing"},{"name":"time to collect","accumulatorId":4035,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4034,"metricType":"sum"},{"name":"data size","accumulatorId":4033,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4116,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4115,"metricType":"timing"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":37,"Submission Time":1784293432262,"Stage Infos":[{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"335\",\"name\":\"WholeStageCodegen (16)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":73,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":76,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"214\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[75],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":69,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"215\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[74,73],"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"357\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"335\",\"name\":\"WholeStageCodegen (16)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432263,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33853","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,/home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.startTime":"1784293421815","spark.rdd.scope":"{\"id\":\"357\",\"name\":\"collect\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784293421026","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33853/jars/org.postgresql_postgresql-42.7.3.jar,spark://73850a3d530b:33853/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_gp","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717170342-0094","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.jars":"file:///home/airflow/.ivy2/jars/org.postgresql_postgresql-42.7.3.jar,file:///home/airflow/.ivy2/jars/org.checkerframework_checker-qual-3.42.0.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33853/files/org.checkerframework_checker-qual-3.42.0.jar,spark://73850a3d530b:33853/files/org.postgresql_postgresql-42.7.3.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":74,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432267,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":74,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784293432267,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784293432345,"Failed":false,"Killed":false,"Accumulables":[{"ID":2951,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2954,"Name":"local bytes read","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2955,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2956,"Name":"records read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4115,"Name":"duration","Update":"41","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4116,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4117,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4118,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4119,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4120,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4121,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4124,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4127,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":4128,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5005654,"Value":5005654,"Internal":true,"Count Failed Values":true},{"ID":4129,"Name":"internal.metrics.executorRunTime","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":4130,"Name":"internal.metrics.executorCpuTime","Update":26629327,"Value":26629327,"Internal":true,"Count Failed Values":true},{"ID":4131,"Name":"internal.metrics.resultSize","Update":4339,"Value":4339,"Internal":true,"Count Failed Values":true},{"ID":4138,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4139,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4140,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4141,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4142,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":4143,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4144,"Name":"internal.metrics.shuffle.read.recordsRead","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4145,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4146,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4147,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4148,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4149,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4150,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4151,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4152,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4153,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4154,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":5005654,"Executor Run Time":59,"Executor CPU Time":26629327,"Peak Execution Memory":0,"Result Size":4339,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":59,"Total Records Read":1,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"335\",\"name\":\"WholeStageCodegen (16)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784293432263,"Completion Time":1784293432346,"Accumulables":[{"ID":2951,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2954,"Name":"local bytes read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2955,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2956,"Name":"records read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4115,"Name":"duration","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4116,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4117,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4118,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4119,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4120,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4121,"Name":"number of output rows","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4124,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4127,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":4128,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5005654,"Internal":true,"Count Failed Values":true},{"ID":4129,"Name":"internal.metrics.executorRunTime","Value":59,"Internal":true,"Count Failed Values":true},{"ID":4130,"Name":"internal.metrics.executorCpuTime","Value":26629327,"Internal":true,"Count Failed Values":true},{"ID":4131,"Name":"internal.metrics.resultSize","Value":4339,"Internal":true,"Count Failed Values":true},{"ID":4138,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4139,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4140,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4141,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4142,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":59,"Internal":true,"Count Failed Values":true},{"ID":4143,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4144,"Name":"internal.metrics.shuffle.read.recordsRead","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4145,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4146,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4147,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4148,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4149,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4150,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4151,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4152,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4153,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4154,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":37,"Completion Time":1784293432346,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1784293432358,"errorMessage":""}
{"Event":"SparkListenerApplicationEnd","Timestamp":1784293432452}