{"Event":"SparkListenerLogStart","Spark Version":"3.5.1"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"73850a3d530b","Port":43147},"Maximum Memory":455501414,"Timestamp":1784265089229,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.19 (Debian)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"73850a3d530b","spark.serializer.objectStreamReset":"100","spark.eventLog.enabled":"true","spark.driver.port":"37165","spark.rdd.compress":"True","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.name":"anazarenko_create_dm_ch","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.submitTime":"1784265087912","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.submit.deployMode":"client","spark.master":"spark://spark-master:7077","spark.eventLog.dir":"file:/opt/spark-events","spark.eventLog.compress":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.id":"app-20260717091129-0090"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n    org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n    org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n  ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"1","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Debian","java.vm.specification.version":"17","user.home":"/home/airflow","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-17-openjdk-amd64/lib","user.dir":"/opt/airflow","java.library.path":"/usr/local/lib:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.19+10-1-deb12u2-Debian","jetty.git.hash":"abdcda73818a1a2c705da276edb0bf6581e7997e","java.runtime.version":"17.0.19+10-1-deb12u2-Debian","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2026-04-21","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Europe/Samara","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.12.48+deb13-amd64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://tracker.debian.org/openjdk-17","os.name":"Linux","java.vm.vendor":"Debian","jdk.debug":"release","java.vendor.url.bug":"https://bugs.debian.org/openjdk-17","user.name":"airflow","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://spark-master:7077 --deploy-mode client --conf spark.eventLog.compress=true --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=file:/opt/spark-events --packages com.clickhouse:clickhouse-jdbc:0.9.0 --name airflow-spark --verbose /opt/airflow/dags/anazarenko/spark/create_dm_ch.py --input s3a://anazarenko-bucket/input/ --output s3a://anazarenko-bucket/output/","java.home":"/usr/lib/jvm/java-17-openjdk-amd64","java.version":"17.0.19","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snappy-java-1.1.10.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JLargeArrays-1.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-logging-1.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/super-csv-2.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zjsonpatch-0.3.0.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar":"Added By User","spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compiler-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sketch_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-common-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-format-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xz-1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-launcher_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-aws-3.3.4.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar":"Added By User","spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill-java-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-server-2.40.jar":"System Classpath","spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/tink-1.9.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-core_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-library-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-metastore-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lz4-java-1.8.0.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/conf":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-tags_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive-thriftserver_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/derby-10.14.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-api_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/gson-2.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/activation-1.1.1.jar":"System Classpath","spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jta-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-mapreduce-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpclient-4.5.14.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill_2.12-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jdo-api-3.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-math3-3.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool2-2.11.1.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/oro-2.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-cli-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-core-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-jdbc-2.3.9.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang3-3.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections4-4.4.jar":"System Classpath","spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/postgresql-42.7.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kubernetes_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-token-provider-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool-1.5.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-column-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jsr305-3.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-json-4.2.19.jar":"System Classpath","spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/threeten-extra-1.7.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar":"Added By User","spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javolution-5.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/shims-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections-3.2.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar":"Added By User","spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jpam-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-shuffle_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/slf4j-api-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libthrift-0.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-repl_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-compiler-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/minlog-1.3.0.jar":"System Classpath","spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar":"Added By User","spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/paranamer-2.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-client-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-vector-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-core-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aircompressor-0.26.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/guava-14.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JTransforms-3.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/py4j-0.10.9.7.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/pickle-1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/opencsv-2.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-databind-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/HikariCP-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ST4-4.0.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-dbcp-1.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-client-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpcore-4.4.16.jar":"System Classpath","spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/joda-time-2.12.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/objenesis-3.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-yarn_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-beeline-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-catalyst_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib-local_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/annotations-17.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aws-java-sdk-bundle-1.12.262.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javassist-3.29.2-GA.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-text-1.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/compress-lzf-1.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-ipc-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-io-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ivy-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-streaming_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okhttp-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-mapred-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-common-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-cli-1.5.0.jar":"System Classpath","spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/clickhouse-jdbc-0.6.0-all.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libfb303-0.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stream-2.9.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-reflect-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-api-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-java-3.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okio-1.15.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-common_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kafka-clients-3.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-common-utils_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lapack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-core-1.9.2-shaded-protobuf.jar":"System Classpath","spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar":"Added By User","spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/janino-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/blas-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-graphx_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jodd-core-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mesos_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar":"Added By User","spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar":"Added By User","spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-hk2-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compress-1.23.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stax-api-1.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-serde-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-all-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-shims-1.9.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/audience-annotations-0.5.0.jar":"System Classpath","spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-framework-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kvstore_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar":"Added By User","spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-codec-1.16.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jline-2.14.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-unsafe_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/transaction-api-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"anazarenko_create_dm_ch","App ID":"app-20260717091129-0090","Timestamp":1784265088616,"User":"airflow"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1784265091157,"Executor ID":"0","Executor Info":{"Host":"172.28.0.49","Total Cores":5,"Log Urls":{"stdout":"http://172.28.0.49:8081/logPage/?appId=app-20260717091129-0090&executorId=0&logType=stdout","stderr":"http://172.28.0.49:8081/logPage/?appId=app-20260717091129-0090&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1784265091156}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.49","Port":37213},"Maximum Memory":455501414,"Timestamp":1784265091210,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (20)\n+- CollectLimit (19)\n   +- HashAggregate (18)\n      +- HashAggregate (17)\n         +- HashAggregate (16)\n            +- HashAggregate (15)\n               +- SortMergeJoin LeftOuter (14)\n                  :- Sort (10)\n                  :  +- Exchange (9)\n                  :     +- Project (8)\n                  :        +- SortMergeJoin Inner (7)\n                  :           :- Sort (3)\n                  :           :  +- Exchange (2)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (6)\n                  :              +- Exchange (5)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (4)\n                  +- Sort (13)\n                     +- Exchange (12)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (11)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(3) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(5) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(6) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(9) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(10) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(12) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(13) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(15) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(16) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(17) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(18) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(19) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(20) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":97,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":91,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":98,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":92,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":89,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":86,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":95,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":88,"metricType":"sum"},{"name":"records read","accumulatorId":85,"metricType":"sum"},{"name":"local bytes read","accumulatorId":83,"metricType":"size"},{"name":"fetch wait time","accumulatorId":84,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":81,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":87,"metricType":"sum"},{"name":"local blocks read","accumulatorId":80,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":90,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":79,"metricType":"sum"},{"name":"data size","accumulatorId":77,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":93,"metricType":"size"},{"name":"number of partitions","accumulatorId":78,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":94,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":82,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":96,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":74,"metricType":"timing"},{"name":"peak memory","accumulatorId":75,"metricType":"size"},{"name":"spill size","accumulatorId":76,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":123,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":117,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":124,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":118,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":115,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":112,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":121,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":114,"metricType":"sum"},{"name":"records read","accumulatorId":111,"metricType":"sum"},{"name":"local bytes read","accumulatorId":109,"metricType":"size"},{"name":"fetch wait time","accumulatorId":110,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":107,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":113,"metricType":"sum"},{"name":"local blocks read","accumulatorId":106,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":116,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":105,"metricType":"sum"},{"name":"data size","accumulatorId":103,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":119,"metricType":"size"},{"name":"number of partitions","accumulatorId":104,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":120,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":108,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":122,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":100,"metricType":"timing"},{"name":"peak memory","accumulatorId":101,"metricType":"size"},{"name":"spill size","accumulatorId":102,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":72,"metricType":"sum"},{"name":"spill size","accumulatorId":73,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":70,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":64,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":71,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":65,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":62,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":59,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":68,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":61,"metricType":"sum"},{"name":"records read","accumulatorId":58,"metricType":"sum"},{"name":"local bytes read","accumulatorId":56,"metricType":"size"},{"name":"fetch wait time","accumulatorId":57,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":54,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":60,"metricType":"sum"},{"name":"local blocks read","accumulatorId":53,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":63,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":52,"metricType":"sum"},{"name":"data size","accumulatorId":50,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":66,"metricType":"size"},{"name":"number of partitions","accumulatorId":51,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":67,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":55,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":69,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":47,"metricType":"timing"},{"name":"peak memory","accumulatorId":48,"metricType":"size"},{"name":"spill size","accumulatorId":49,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":149,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":143,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":150,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":144,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":141,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":138,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":147,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":140,"metricType":"sum"},{"name":"records read","accumulatorId":137,"metricType":"sum"},{"name":"local bytes read","accumulatorId":135,"metricType":"size"},{"name":"fetch wait time","accumulatorId":136,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":133,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":139,"metricType":"sum"},{"name":"local blocks read","accumulatorId":132,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":142,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":131,"metricType":"sum"},{"name":"data size","accumulatorId":129,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":145,"metricType":"size"},{"name":"number of partitions","accumulatorId":130,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":146,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":134,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":148,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":126,"metricType":"timing"},{"name":"peak memory","accumulatorId":127,"metricType":"size"},{"name":"spill size","accumulatorId":128,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":45,"metricType":"sum"},{"name":"spill size","accumulatorId":46,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":41,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":42,"metricType":"timing"},{"name":"peak memory","accumulatorId":40,"metricType":"size"},{"name":"number of output rows","accumulatorId":39,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":44,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":43,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":35,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":36,"metricType":"timing"},{"name":"peak memory","accumulatorId":34,"metricType":"size"},{"name":"number of output rows","accumulatorId":33,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":38,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":37,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":29,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":30,"metricType":"timing"},{"name":"peak memory","accumulatorId":28,"metricType":"size"},{"name":"number of output rows","accumulatorId":27,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":32,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":31,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":23,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":24,"metricType":"timing"},{"name":"peak memory","accumulatorId":22,"metricType":"size"},{"name":"number of output rows","accumulatorId":21,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":26,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":25,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":19,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":13,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":20,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":14,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":11,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":8,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":17,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":10,"metricType":"sum"},{"name":"records read","accumulatorId":7,"metricType":"sum"},{"name":"local bytes read","accumulatorId":5,"metricType":"size"},{"name":"fetch wait time","accumulatorId":6,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":9,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":12,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":15,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":16,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":18,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784265091707,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (39)\n+- == Current Plan ==\n   CollectLimit (22)\n   +- HashAggregate (21)\n      +- HashAggregate (20)\n         +- HashAggregate (19)\n            +- HashAggregate (18)\n               +- SortMergeJoin LeftOuter (17)\n                  :- Sort (12)\n                  :  +- Exchange (11)\n                  :     +- Project (10)\n                  :        +- SortMergeJoin Inner (9)\n                  :           :- Sort (4)\n                  :           :  +- ShuffleQueryStage (3)\n                  :           :     +- Exchange (2)\n                  :           :        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (8)\n                  :              +- ShuffleQueryStage (7)\n                  :                 +- Exchange (6)\n                  :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (16)\n                     +- ShuffleQueryStage (15)\n                        +- Exchange (14)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   CollectLimit (38)\n   +- HashAggregate (37)\n      +- HashAggregate (36)\n         +- HashAggregate (35)\n            +- HashAggregate (34)\n               +- SortMergeJoin LeftOuter (33)\n                  :- Sort (30)\n                  :  +- Exchange (29)\n                  :     +- Project (28)\n                  :        +- SortMergeJoin Inner (27)\n                  :           :- Sort (24)\n                  :           :  +- Exchange (23)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (26)\n                  :              +- Exchange (25)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (32)\n                     +- Exchange (31)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(6) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(7) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(8) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(11) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=91]\n\n(12) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(14) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(15) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(16) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(18) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(19) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(20) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(21) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(22) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(23) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(24) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(25) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(26) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(27) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(28) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(29) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(30) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(31) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(32) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(33) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(34) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(35) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(36) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(37) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(38) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(39) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=91]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":225,"metricType":"timing"},{"name":"peak memory","accumulatorId":226,"metricType":"size"},{"name":"spill size","accumulatorId":227,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":251,"metricType":"timing"},{"name":"peak memory","accumulatorId":252,"metricType":"size"},{"name":"spill size","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":223,"metricType":"sum"},{"name":"spill size","accumulatorId":224,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":221,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":215,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":222,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":216,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":213,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":210,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":219,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":212,"metricType":"sum"},{"name":"records read","accumulatorId":209,"metricType":"sum"},{"name":"local bytes read","accumulatorId":207,"metricType":"size"},{"name":"fetch wait time","accumulatorId":208,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":205,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":211,"metricType":"sum"},{"name":"local blocks read","accumulatorId":204,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":214,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":203,"metricType":"sum"},{"name":"data size","accumulatorId":201,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":217,"metricType":"size"},{"name":"number of partitions","accumulatorId":202,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":218,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":206,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":220,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":198,"metricType":"timing"},{"name":"peak memory","accumulatorId":199,"metricType":"size"},{"name":"spill size","accumulatorId":200,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":277,"metricType":"timing"},{"name":"peak memory","accumulatorId":278,"metricType":"size"},{"name":"spill size","accumulatorId":279,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":196,"metricType":"sum"},{"name":"spill size","accumulatorId":197,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":192,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":193,"metricType":"timing"},{"name":"peak memory","accumulatorId":191,"metricType":"size"},{"name":"number of output rows","accumulatorId":190,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":195,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":194,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":186,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":187,"metricType":"timing"},{"name":"peak memory","accumulatorId":185,"metricType":"size"},{"name":"number of output rows","accumulatorId":184,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":189,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":188,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":180,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":181,"metricType":"timing"},{"name":"peak memory","accumulatorId":179,"metricType":"size"},{"name":"number of output rows","accumulatorId":178,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":183,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":182,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":174,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":175,"metricType":"timing"},{"name":"peak memory","accumulatorId":173,"metricType":"size"},{"name":"number of output rows","accumulatorId":172,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":177,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":176,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":170,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":164,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":171,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":165,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":162,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":159,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":168,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":161,"metricType":"sum"},{"name":"records read","accumulatorId":158,"metricType":"sum"},{"name":"local bytes read","accumulatorId":156,"metricType":"size"},{"name":"fetch wait time","accumulatorId":157,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":154,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":160,"metricType":"sum"},{"name":"local blocks read","accumulatorId":153,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":163,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":152,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":166,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":167,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":155,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":169,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[229,200]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[255,200]]}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1784265092025,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092036,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[281,200]]}
{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1784265092159,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092160,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1784265092178,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092179,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092162,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092181,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092191,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092181,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265093287,"Failed":false,"Killed":false,"Accumulables":[{"ID":125,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":254,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":273,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":274,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":275,"Name":"shuffle write time","Update":"17916479","Value":"17916479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":276,"Name":"duration","Update":"129","Value":"129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"internal.metrics.executorDeserializeTime","Update":289,"Value":289,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.executorDeserializeCpuTime","Update":154943081,"Value":154943081,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.executorRunTime","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.executorCpuTime","Update":338674547,"Value":338674547,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":343,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":344,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.write.writeTime","Update":17916479,"Value":17916479,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":289,"Executor Deserialize CPU Time":154943081,"Executor Run Time":777,"Executor CPU Time":338674547,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":21,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":17916479,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092160,"Completion Time":1784265093298,"Accumulables":[{"ID":125,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":254,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":273,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":274,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":275,"Name":"shuffle write time","Value":"17916479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":276,"Name":"duration","Value":"129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"internal.metrics.executorDeserializeTime","Value":289,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.executorDeserializeCpuTime","Value":154943081,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.executorRunTime","Value":777,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.executorCpuTime","Value":338674547,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":343,"Name":"internal.metrics.jvmGCTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":344,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.write.writeTime","Value":17916479,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1784265093310,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092162,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265093289,"Failed":false,"Killed":false,"Accumulables":[{"ID":99,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":228,"Name":"data size","Update":"1648","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":247,"Name":"shuffle bytes written","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":248,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":249,"Name":"shuffle write time","Update":"18323280","Value":"18323280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":250,"Name":"duration","Update":"129","Value":"129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":303,"Name":"internal.metrics.executorDeserializeTime","Update":289,"Value":289,"Internal":true,"Count Failed Values":true},{"ID":304,"Name":"internal.metrics.executorDeserializeCpuTime","Update":136161428,"Value":136161428,"Internal":true,"Count Failed Values":true},{"ID":305,"Name":"internal.metrics.executorRunTime","Update":776,"Value":776,"Internal":true,"Count Failed Values":true},{"ID":306,"Name":"internal.metrics.executorCpuTime","Update":303174773,"Value":303174773,"Internal":true,"Count Failed Values":true},{"ID":307,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":308,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":309,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1210,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":333,"Name":"internal.metrics.shuffle.write.writeTime","Update":18323280,"Value":18323280,"Internal":true,"Count Failed Values":true},{"ID":335,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":289,"Executor Deserialize CPU Time":136161428,"Executor Run Time":776,"Executor CPU Time":303174773,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":21,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1210,"Shuffle Write Time":18323280,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092036,"Completion Time":1784265093314,"Accumulables":[{"ID":99,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":228,"Name":"data size","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":247,"Name":"shuffle bytes written","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":248,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":249,"Name":"shuffle write time","Value":"18323280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":250,"Name":"duration","Value":"129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":303,"Name":"internal.metrics.executorDeserializeTime","Value":289,"Internal":true,"Count Failed Values":true},{"ID":304,"Name":"internal.metrics.executorDeserializeCpuTime","Value":136161428,"Internal":true,"Count Failed Values":true},{"ID":305,"Name":"internal.metrics.executorRunTime","Value":776,"Internal":true,"Count Failed Values":true},{"ID":306,"Name":"internal.metrics.executorCpuTime","Value":303174773,"Internal":true,"Count Failed Values":true},{"ID":307,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":308,"Name":"internal.metrics.jvmGCTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":309,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1210,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":333,"Name":"internal.metrics.shuffle.write.writeTime","Value":18323280,"Internal":true,"Count Failed Values":true},{"ID":335,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1784265093320,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (42)\n+- == Current Plan ==\n   CollectLimit (25)\n   +- HashAggregate (24)\n      +- HashAggregate (23)\n         +- HashAggregate (22)\n            +- HashAggregate (21)\n               +- SortMergeJoin LeftOuter (20)\n                  :- Sort (15)\n                  :  +- ShuffleQueryStage (14)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (19)\n                     +- ShuffleQueryStage (18)\n                        +- Exchange (17)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   CollectLimit (41)\n   +- HashAggregate (40)\n      +- HashAggregate (39)\n         +- HashAggregate (38)\n            +- HashAggregate (37)\n               +- SortMergeJoin LeftOuter (36)\n                  :- Sort (33)\n                  :  +- Exchange (32)\n                  :     +- Project (31)\n                  :        +- SortMergeJoin Inner (30)\n                  :           :- Sort (27)\n                  :           :  +- Exchange (26)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (29)\n                  :              +- Exchange (28)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (35)\n                     +- Exchange (34)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(20) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(21) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(22) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(25) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(26) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(27) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(28) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(29) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(31) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(32) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(33) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(34) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(35) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(37) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(38) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(41) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(42) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":454,"metricType":"timing"},{"name":"peak memory","accumulatorId":455,"metricType":"size"},{"name":"spill size","accumulatorId":456,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":496,"metricType":"timing"},{"name":"peak memory","accumulatorId":497,"metricType":"size"},{"name":"spill size","accumulatorId":498,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"},{"name":"spill size","accumulatorId":453,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":448,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":449,"metricType":"timing"},{"name":"peak memory","accumulatorId":447,"metricType":"size"},{"name":"number of output rows","accumulatorId":446,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":451,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":450,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":442,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":443,"metricType":"timing"},{"name":"peak memory","accumulatorId":441,"metricType":"size"},{"name":"number of output rows","accumulatorId":440,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":445,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":444,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":436,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":437,"metricType":"timing"},{"name":"peak memory","accumulatorId":435,"metricType":"size"},{"name":"number of output rows","accumulatorId":434,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":439,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":438,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":430,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":431,"metricType":"timing"},{"name":"peak memory","accumulatorId":429,"metricType":"size"},{"name":"number of output rows","accumulatorId":428,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":433,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":432,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":426,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":420,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":427,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":421,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":418,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":415,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":424,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":417,"metricType":"sum"},{"name":"records read","accumulatorId":414,"metricType":"sum"},{"name":"local bytes read","accumulatorId":412,"metricType":"size"},{"name":"fetch wait time","accumulatorId":413,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":410,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":416,"metricType":"sum"},{"name":"local blocks read","accumulatorId":409,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":419,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":408,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":422,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":423,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":411,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":425,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[486,1],[488,1],[487,1287]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[493,1],[495,1],[494,578]]}
{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265092191,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265093492,"Failed":false,"Killed":false,"Accumulables":[{"ID":151,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":280,"Name":"data size","Update":"3648","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":299,"Name":"shuffle bytes written","Update":"2066","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"shuffle records written","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":301,"Name":"shuffle write time","Update":"4802951","Value":"4802951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"duration","Update":"365","Value":"365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"internal.metrics.executorDeserializeTime","Update":288,"Value":288,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.executorDeserializeCpuTime","Update":131088462,"Value":131088462,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.executorRunTime","Update":998,"Value":998,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.executorCpuTime","Update":529483812,"Value":529483812,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.resultSize","Update":1983,"Value":1983,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.jvmGCTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":401,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2066,"Value":2066,"Internal":true,"Count Failed Values":true},{"ID":402,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":403,"Name":"internal.metrics.shuffle.write.writeTime","Update":4802951,"Value":4802951,"Internal":true,"Count Failed Values":true},{"ID":405,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":288,"Executor Deserialize CPU Time":131088462,"Executor Run Time":998,"Executor CPU Time":529483812,"Peak Execution Memory":0,"Result Size":1983,"JVM GC Time":21,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2066,"Shuffle Write Time":4802951,"Shuffle Records Written":57},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265092179,"Completion Time":1784265093493,"Accumulables":[{"ID":151,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":280,"Name":"data size","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":299,"Name":"shuffle bytes written","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"shuffle records written","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":301,"Name":"shuffle write time","Value":"4802951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"duration","Value":"365","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"internal.metrics.executorDeserializeTime","Value":288,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.executorDeserializeCpuTime","Value":131088462,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.executorRunTime","Value":998,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.executorCpuTime","Value":529483812,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.resultSize","Value":1983,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.jvmGCTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":401,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2066,"Internal":true,"Count Failed Values":true},{"ID":402,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":57,"Internal":true,"Count Failed Values":true},{"ID":403,"Name":"internal.metrics.shuffle.write.writeTime","Value":4802951,"Internal":true,"Count Failed Values":true},{"ID":405,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1784265093495,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[458,200]]}
{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1784265093515,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5,3,4],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265093526,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265093546,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- == Current Plan ==\n   CollectLimit (26)\n   +- HashAggregate (25)\n      +- HashAggregate (24)\n         +- HashAggregate (23)\n            +- HashAggregate (22)\n               +- BroadcastHashJoin LeftOuter BuildRight (21)\n                  :- ShuffleQueryStage (14)\n                  :  +- Exchange (13)\n                  :     +- * Project (12)\n                  :        +- * SortMergeJoin Inner (11)\n                  :           :- * Sort (5)\n                  :           :  +- AQEShuffleRead (4)\n                  :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :           :        +- Exchange (2)\n                  :           :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- * Sort (10)\n                  :              +- AQEShuffleRead (9)\n                  :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                    +- Exchange (7)\n                  :                       +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (20)\n                     +- BroadcastExchange (19)\n                        +- AQEShuffleRead (18)\n                           +- ShuffleQueryStage (17), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (16)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (42)\n   +- HashAggregate (41)\n      +- HashAggregate (40)\n         +- HashAggregate (39)\n            +- HashAggregate (38)\n               +- SortMergeJoin LeftOuter (37)\n                  :- Sort (34)\n                  :  +- Exchange (33)\n                  :     +- Project (32)\n                  :        +- SortMergeJoin Inner (31)\n                  :           :- Sort (28)\n                  :           :  +- Exchange (27)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (30)\n                  :              +- Exchange (29)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (36)\n                     +- Exchange (35)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(16) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(17) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(19) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]\n\n(20) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(21) BroadcastHashJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(25) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(26) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(27) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(28) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(30) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(32) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(33) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(34) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(35) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(36) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(42) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(43) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":583,"metricType":"timing"},{"name":"time to build","accumulatorId":582,"metricType":"timing"},{"name":"time to collect","accumulatorId":581,"metricType":"timing"},{"name":"number of output rows","accumulatorId":580,"metricType":"sum"},{"name":"data size","accumulatorId":579,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":578,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":574,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":575,"metricType":"timing"},{"name":"peak memory","accumulatorId":573,"metricType":"size"},{"name":"number of output rows","accumulatorId":572,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":577,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":576,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":562,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":563,"metricType":"timing"},{"name":"peak memory","accumulatorId":561,"metricType":"size"},{"name":"number of output rows","accumulatorId":560,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":565,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":564,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":556,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":557,"metricType":"timing"},{"name":"peak memory","accumulatorId":555,"metricType":"size"},{"name":"number of output rows","accumulatorId":554,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":559,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":558,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":552,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":546,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":553,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":547,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":544,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":541,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":550,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":543,"metricType":"sum"},{"name":"records read","accumulatorId":540,"metricType":"sum"},{"name":"local bytes read","accumulatorId":538,"metricType":"size"},{"name":"fetch wait time","accumulatorId":539,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":536,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":542,"metricType":"sum"},{"name":"local blocks read","accumulatorId":535,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":545,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":534,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":548,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":549,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":537,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":551,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[584,1]]}
{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1784265093599,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[6,7],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId bac39180-2cd4-4465-8b4a-599405da1e01)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265093607,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId bac39180-2cd4-4465-8b4a-599405da1e01)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265093625,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265093625,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265093769,"Failed":false,"Killed":false,"Accumulables":[{"ID":283,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":286,"Name":"local bytes read","Update":"2066","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":287,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"records read","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":585,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":586,"Name":"internal.metrics.executorDeserializeCpuTime","Update":55939760,"Value":55939760,"Internal":true,"Count Failed Values":true},{"ID":587,"Name":"internal.metrics.executorRunTime","Update":56,"Value":56,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.executorCpuTime","Update":36814410,"Value":36814410,"Internal":true,"Count Failed Values":true},{"ID":589,"Name":"internal.metrics.resultSize","Update":4604,"Value":4604,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2066,"Value":2066,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.shuffle.read.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":604,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":605,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":609,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":55939760,"Executor Run Time":56,"Executor CPU Time":36814410,"Peak Execution Memory":0,"Result Size":4604,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2066,"Total Records Read":57,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265093607,"Completion Time":1784265093774,"Accumulables":[{"ID":283,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":286,"Name":"local bytes read","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":287,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"records read","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":585,"Name":"internal.metrics.executorDeserializeTime","Value":71,"Internal":true,"Count Failed Values":true},{"ID":586,"Name":"internal.metrics.executorDeserializeCpuTime","Value":55939760,"Internal":true,"Count Failed Values":true},{"ID":587,"Name":"internal.metrics.executorRunTime","Value":56,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.executorCpuTime","Value":36814410,"Internal":true,"Count Failed Values":true},{"ID":589,"Name":"internal.metrics.resultSize","Value":4604,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2066,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.shuffle.read.recordsRead","Value":57,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":604,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":605,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":609,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1784265093779,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[583,12],[582,42],[581,211],[580,57],[579,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265093546,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265093938,"Failed":false,"Killed":false,"Accumulables":[{"ID":231,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":234,"Name":"local bytes read","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":235,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":257,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":260,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":261,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":262,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":457,"Name":"data size","Update":"2456","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":476,"Name":"shuffle bytes written","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":477,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":478,"Name":"shuffle write time","Update":"4536319","Value":"4536319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":479,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":480,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":481,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"duration","Update":"150","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":483,"Name":"sort time","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":484,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":485,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"duration","Update":"64","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":490,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":492,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"internal.metrics.executorDeserializeTime","Update":126,"Value":126,"Internal":true,"Count Failed Values":true},{"ID":500,"Name":"internal.metrics.executorDeserializeCpuTime","Update":89941203,"Value":89941203,"Internal":true,"Count Failed Values":true},{"ID":501,"Name":"internal.metrics.executorRunTime","Update":249,"Value":249,"Internal":true,"Count Failed Values":true},{"ID":502,"Name":"internal.metrics.executorCpuTime","Update":184894643,"Value":184894643,"Internal":true,"Count Failed Values":true},{"ID":503,"Name":"internal.metrics.resultSize","Update":6375,"Value":6375,"Internal":true,"Count Failed Values":true},{"ID":508,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":514,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1764,"Value":1764,"Internal":true,"Count Failed Values":true},{"ID":515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":516,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":517,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":518,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":520,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":521,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":522,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":524,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":525,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":526,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":527,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":528,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":529,"Name":"internal.metrics.shuffle.write.writeTime","Update":4536319,"Value":4536319,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":126,"Executor Deserialize CPU Time":89941203,"Executor Run Time":249,"Executor CPU Time":184894643,"Peak Execution Memory":8519648,"Result Size":6375,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1764,"Total Records Read":23,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2786,"Shuffle Write Time":4536319,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265093526,"Completion Time":1784265093940,"Accumulables":[{"ID":231,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":234,"Name":"local bytes read","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":235,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":257,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":260,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":261,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":262,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":457,"Name":"data size","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":476,"Name":"shuffle bytes written","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":477,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":478,"Name":"shuffle write time","Value":"4536319","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":479,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":480,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":481,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"duration","Value":"150","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":483,"Name":"sort time","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":484,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":485,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"duration","Value":"64","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":490,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":492,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"internal.metrics.executorDeserializeTime","Value":126,"Internal":true,"Count Failed Values":true},{"ID":500,"Name":"internal.metrics.executorDeserializeCpuTime","Value":89941203,"Internal":true,"Count Failed Values":true},{"ID":501,"Name":"internal.metrics.executorRunTime","Value":249,"Internal":true,"Count Failed Values":true},{"ID":502,"Name":"internal.metrics.executorCpuTime","Value":184894643,"Internal":true,"Count Failed Values":true},{"ID":503,"Name":"internal.metrics.resultSize","Value":6375,"Internal":true,"Count Failed Values":true},{"ID":508,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":514,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1764,"Internal":true,"Count Failed Values":true},{"ID":515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":516,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true},{"ID":517,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":518,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":520,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":521,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":522,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":524,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":525,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":526,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":527,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":528,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":529,"Name":"internal.metrics.shuffle.write.writeTime","Value":4536319,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1784265093942,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Final Plan ==\n   CollectLimit (27)\n   +- * HashAggregate (26)\n      +- * HashAggregate (25)\n         +- * HashAggregate (24)\n            +- * HashAggregate (23)\n               +- * BroadcastHashJoin LeftOuter BuildRight (22)\n                  :- AQEShuffleRead (15)\n                  :  +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.4 KiB, rowCount=18)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (21), Statistics(sizeInBytes=2.0 MiB, rowCount=57)\n                     +- BroadcastExchange (20)\n                        +- AQEShuffleRead (19)\n                           +- ShuffleQueryStage (18), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (17)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   CollectLimit (43)\n   +- HashAggregate (42)\n      +- HashAggregate (41)\n         +- HashAggregate (40)\n            +- HashAggregate (39)\n               +- SortMergeJoin LeftOuter (38)\n                  :- Sort (35)\n                  :  +- Exchange (34)\n                  :     +- Project (33)\n                  :        +- SortMergeJoin Inner (32)\n                  :           :- Sort (29)\n                  :           :  +- Exchange (28)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (31)\n                  :              +- Exchange (30)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (37)\n                     +- Exchange (36)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) AQEShuffleRead\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: local\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(20) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]\n\n(21) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(22) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) HashAggregate [codegen id : 7]\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(25) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(26) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(27) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(28) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(29) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(30) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(31) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(33) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(34) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(35) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(36) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(37) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(42) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(43) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(44) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":666,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":583,"metricType":"timing"},{"name":"time to build","accumulatorId":582,"metricType":"timing"},{"name":"time to collect","accumulatorId":581,"metricType":"timing"},{"name":"number of output rows","accumulatorId":580,"metricType":"sum"},{"name":"data size","accumulatorId":579,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":665,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":661,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":662,"metricType":"timing"},{"name":"peak memory","accumulatorId":660,"metricType":"size"},{"name":"number of output rows","accumulatorId":659,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":664,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":663,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":655,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":656,"metricType":"timing"},{"name":"peak memory","accumulatorId":654,"metricType":"size"},{"name":"number of output rows","accumulatorId":653,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":658,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":657,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":649,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":650,"metricType":"timing"},{"name":"peak memory","accumulatorId":648,"metricType":"size"},{"name":"number of output rows","accumulatorId":647,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":652,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":651,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":643,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":644,"metricType":"timing"},{"name":"peak memory","accumulatorId":642,"metricType":"size"},{"name":"number of output rows","accumulatorId":641,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":646,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":645,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":640,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":638,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":632,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":639,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":633,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":630,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":627,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":636,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":629,"metricType":"sum"},{"name":"records read","accumulatorId":626,"metricType":"sum"},{"name":"local bytes read","accumulatorId":624,"metricType":"size"},{"name":"fetch wait time","accumulatorId":625,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":622,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":628,"metricType":"sum"},{"name":"local blocks read","accumulatorId":621,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":631,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":620,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":634,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":635,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":623,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":637,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[666,1]]}
{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1784265094134,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[9,8],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[9,10,11,8],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265094135,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265094143,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265094143,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265094463,"Failed":false,"Killed":false,"Accumulables":[{"ID":460,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":463,"Name":"local bytes read","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":464,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":465,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":640,"Name":"duration","Update":"117","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":641,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":642,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":644,"Name":"time in aggregation build","Update":"66","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":648,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":650,"Name":"time in aggregation build","Update":"51","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":653,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":654,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":656,"Name":"time in aggregation build","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":657,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":659,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":660,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":662,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":663,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":665,"Name":"number of output rows","Update":"60","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":667,"Name":"internal.metrics.executorDeserializeTime","Update":84,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":668,"Name":"internal.metrics.executorDeserializeCpuTime","Update":78411280,"Value":78411280,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorRunTime","Update":227,"Value":227,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorCpuTime","Update":220041891,"Value":220041891,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.resultSize","Update":12208,"Value":12208,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.peakExecutionMemory","Update":17826752,"Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":678,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":679,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":680,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":681,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":682,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":683,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":684,"Name":"internal.metrics.shuffle.read.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":685,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":686,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":687,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":688,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":689,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":690,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":691,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":692,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":693,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":694,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":84,"Executor Deserialize CPU Time":78411280,"Executor Run Time":227,"Executor CPU Time":220041891,"Peak Execution Memory":17826752,"Result Size":12208,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2786,"Total Records Read":18,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265094135,"Completion Time":1784265094464,"Accumulables":[{"ID":460,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":463,"Name":"local bytes read","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":464,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":465,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":640,"Name":"duration","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":641,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":642,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":644,"Name":"time in aggregation build","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":648,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":650,"Name":"time in aggregation build","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":653,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":654,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":656,"Name":"time in aggregation build","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":657,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":659,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":660,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":662,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":663,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":665,"Name":"number of output rows","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":667,"Name":"internal.metrics.executorDeserializeTime","Value":84,"Internal":true,"Count Failed Values":true},{"ID":668,"Name":"internal.metrics.executorDeserializeCpuTime","Value":78411280,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorRunTime","Value":227,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorCpuTime","Value":220041891,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.resultSize","Value":12208,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.peakExecutionMemory","Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":678,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":679,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":680,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":681,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":682,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":683,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":684,"Name":"internal.metrics.shuffle.read.recordsRead","Value":18,"Internal":true,"Count Failed Values":true},{"ID":685,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":686,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":687,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":688,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":689,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":690,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":691,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":692,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":693,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":694,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1784265094464,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1784265095012,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (22)\n+- CollectLimit (21)\n   +- Project (20)\n      +- SortMergeJoin LeftOuter (19)\n         :- LocalLimit (10)\n         :  +- SortMergeJoin LeftAnti (9)\n         :     :- Sort (3)\n         :     :  +- Exchange (2)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (8)\n         :        +- HashAggregate (7)\n         :           +- Exchange (6)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (18)\n            +- HashAggregate (17)\n               +- Exchange (16)\n                  +- HashAggregate (15)\n                     +- HashAggregate (14)\n                        +- Exchange (13)\n                           +- HashAggregate (12)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (11)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(3) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(7) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(8) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(10) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(11) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(12) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(13) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(14) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(16) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(17) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(18) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(19) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(20) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(21) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(22) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":749,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":743,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":750,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":744,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":741,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":738,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":747,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":740,"metricType":"sum"},{"name":"records read","accumulatorId":737,"metricType":"sum"},{"name":"local bytes read","accumulatorId":735,"metricType":"size"},{"name":"fetch wait time","accumulatorId":736,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":733,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":739,"metricType":"sum"},{"name":"local blocks read","accumulatorId":732,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":742,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":731,"metricType":"sum"},{"name":"data size","accumulatorId":729,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":745,"metricType":"size"},{"name":"number of partitions","accumulatorId":730,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":746,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":734,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":748,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":726,"metricType":"timing"},{"name":"peak memory","accumulatorId":727,"metricType":"size"},{"name":"spill size","accumulatorId":728,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":781,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":775,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":782,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":776,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":773,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":770,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":779,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":772,"metricType":"sum"},{"name":"records read","accumulatorId":769,"metricType":"sum"},{"name":"local bytes read","accumulatorId":767,"metricType":"size"},{"name":"fetch wait time","accumulatorId":768,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":765,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":771,"metricType":"sum"},{"name":"local blocks read","accumulatorId":764,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":774,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":763,"metricType":"sum"},{"name":"data size","accumulatorId":761,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":777,"metricType":"size"},{"name":"number of partitions","accumulatorId":762,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":778,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":766,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":780,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":757,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":758,"metricType":"timing"},{"name":"peak memory","accumulatorId":756,"metricType":"size"},{"name":"number of output rows","accumulatorId":755,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":760,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":759,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":752,"metricType":"timing"},{"name":"peak memory","accumulatorId":753,"metricType":"size"},{"name":"spill size","accumulatorId":754,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":724,"metricType":"sum"},{"name":"spill size","accumulatorId":725,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":853,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":847,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":854,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":848,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":845,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":842,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":851,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":844,"metricType":"sum"},{"name":"records read","accumulatorId":841,"metricType":"sum"},{"name":"local bytes read","accumulatorId":839,"metricType":"size"},{"name":"fetch wait time","accumulatorId":840,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":837,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":843,"metricType":"sum"},{"name":"local blocks read","accumulatorId":836,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":846,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":835,"metricType":"sum"},{"name":"data size","accumulatorId":833,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":849,"metricType":"size"},{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":850,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":838,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":852,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":829,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":830,"metricType":"timing"},{"name":"peak memory","accumulatorId":828,"metricType":"size"},{"name":"number of output rows","accumulatorId":827,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":832,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":831,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":823,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":824,"metricType":"timing"},{"name":"peak memory","accumulatorId":822,"metricType":"size"},{"name":"number of output rows","accumulatorId":821,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":826,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":825,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":795,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":796,"metricType":"timing"},{"name":"peak memory","accumulatorId":794,"metricType":"size"},{"name":"number of output rows","accumulatorId":793,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":798,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":797,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":790,"metricType":"timing"},{"name":"peak memory","accumulatorId":791,"metricType":"size"},{"name":"spill size","accumulatorId":792,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":722,"metricType":"sum"},{"name":"spill size","accumulatorId":723,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":720,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":714,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":721,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":715,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":712,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":709,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":718,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":711,"metricType":"sum"},{"name":"records read","accumulatorId":708,"metricType":"sum"},{"name":"local bytes read","accumulatorId":706,"metricType":"size"},{"name":"fetch wait time","accumulatorId":707,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":704,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":710,"metricType":"sum"},{"name":"local blocks read","accumulatorId":703,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":713,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":702,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":716,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":717,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":705,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":719,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784265095068,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (41)\n+- == Current Plan ==\n   CollectLimit (24)\n   +- Project (23)\n      +- SortMergeJoin LeftOuter (22)\n         :- LocalLimit (12)\n         :  +- SortMergeJoin LeftAnti (11)\n         :     :- Sort (4)\n         :     :  +- ShuffleQueryStage (3)\n         :     :     +- Exchange (2)\n         :     :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (10)\n         :        +- HashAggregate (9)\n         :           +- ShuffleQueryStage (8)\n         :              +- Exchange (7)\n         :                 +- * HashAggregate (6)\n         :                    +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (21)\n            +- HashAggregate (20)\n               +- Exchange (19)\n                  +- HashAggregate (18)\n                     +- HashAggregate (17)\n                        +- ShuffleQueryStage (16)\n                           +- Exchange (15)\n                              +- * HashAggregate (14)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   CollectLimit (40)\n   +- Project (39)\n      +- SortMergeJoin LeftOuter (38)\n         :- LocalLimit (31)\n         :  +- SortMergeJoin LeftAnti (30)\n         :     :- Sort (26)\n         :     :  +- Exchange (25)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (29)\n         :        +- HashAggregate (28)\n         :           +- Exchange (27)\n         :              +- HashAggregate (6)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (37)\n            +- HashAggregate (36)\n               +- Exchange (35)\n                  +- HashAggregate (34)\n                     +- HashAggregate (33)\n                        +- Exchange (32)\n                           +- HashAggregate (14)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(12) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(13) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(14) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(17) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(18) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(19) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]\n\n(20) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(21) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(22) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(24) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(25) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(26) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(27) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(28) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(29) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(31) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(32) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(33) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(34) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(35) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(36) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(37) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(40) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(41) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":886,"metricType":"timing"},{"name":"peak memory","accumulatorId":887,"metricType":"size"},{"name":"spill size","accumulatorId":888,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":917,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":918,"metricType":"timing"},{"name":"peak memory","accumulatorId":916,"metricType":"size"},{"name":"number of output rows","accumulatorId":915,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":920,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":919,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":912,"metricType":"timing"},{"name":"peak memory","accumulatorId":913,"metricType":"size"},{"name":"spill size","accumulatorId":914,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":884,"metricType":"sum"},{"name":"spill size","accumulatorId":885,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":983,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":984,"metricType":"timing"},{"name":"peak memory","accumulatorId":982,"metricType":"size"},{"name":"number of output rows","accumulatorId":981,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":986,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":985,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":977,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":978,"metricType":"timing"},{"name":"peak memory","accumulatorId":976,"metricType":"size"},{"name":"number of output rows","accumulatorId":975,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":980,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":979,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":973,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":967,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":974,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":968,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":965,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":962,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":971,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":964,"metricType":"sum"},{"name":"records read","accumulatorId":961,"metricType":"sum"},{"name":"local bytes read","accumulatorId":959,"metricType":"size"},{"name":"fetch wait time","accumulatorId":960,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":957,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":963,"metricType":"sum"},{"name":"local blocks read","accumulatorId":956,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":966,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":955,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":969,"metricType":"size"},{"name":"number of partitions","accumulatorId":954,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":970,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":958,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":972,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":949,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":950,"metricType":"timing"},{"name":"peak memory","accumulatorId":948,"metricType":"size"},{"name":"number of output rows","accumulatorId":947,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":952,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":951,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":944,"metricType":"timing"},{"name":"peak memory","accumulatorId":945,"metricType":"size"},{"name":"spill size","accumulatorId":946,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":882,"metricType":"sum"},{"name":"spill size","accumulatorId":883,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":880,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":874,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":881,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":875,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":872,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":869,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":878,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":871,"metricType":"sum"},{"name":"records read","accumulatorId":868,"metricType":"sum"},{"name":"local bytes read","accumulatorId":866,"metricType":"size"},{"name":"fetch wait time","accumulatorId":867,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":864,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":870,"metricType":"sum"},{"name":"local blocks read","accumulatorId":863,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":873,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":862,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":876,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":877,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":865,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":879,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[890,200]]}
{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1784265095093,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[12],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095094,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095104,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[922,200]]}
{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1784265095124,"Stage Infos":[{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[13],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095125,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":13,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095131,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[988,200]]}
{"Event":"SparkListenerJobStart","Job ID":8,"Submission Time":1784265095155,"Stage Infos":[{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[14],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095156,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":14,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095162,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095104,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095192,"Failed":false,"Killed":false,"Accumulables":[{"ID":751,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":889,"Name":"data size","Update":"1760","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"shuffle bytes written","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":909,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":910,"Name":"shuffle write time","Update":"4026374","Value":"4026374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":911,"Name":"duration","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1010,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8337980,"Value":8337980,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.executorRunTime","Update":66,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.executorCpuTime","Update":34308564,"Value":34308564,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.write.writeTime","Update":4026374,"Value":4026374,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":8337980,"Executor Run Time":66,"Executor CPU Time":34308564,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2299,"Shuffle Write Time":4026374,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095094,"Completion Time":1784265095197,"Accumulables":[{"ID":751,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":889,"Name":"data size","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"shuffle bytes written","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":909,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":910,"Name":"shuffle write time","Value":"4026374","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":911,"Name":"duration","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1010,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8337980,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.executorRunTime","Value":66,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.executorCpuTime","Value":34308564,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.write.writeTime","Value":4026374,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1784265095198,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":13,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095131,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095236,"Failed":false,"Killed":false,"Accumulables":[{"ID":783,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":784,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":789,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":921,"Name":"data size","Update":"352","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"shuffle bytes written","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":941,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":942,"Name":"shuffle write time","Update":"4719440","Value":"4719440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":943,"Name":"duration","Update":"19","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9431660,"Value":9431660,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Update":81,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Update":53455371,"Value":53455371,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.shuffle.write.writeTime","Update":4719440,"Value":4719440,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":9431660,"Executor Run Time":81,"Executor CPU Time":53455371,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":777,"Shuffle Write Time":4719440,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095125,"Completion Time":1784265095237,"Accumulables":[{"ID":783,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":784,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":789,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":921,"Name":"data size","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"shuffle bytes written","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":941,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":942,"Name":"shuffle write time","Value":"4719440","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":943,"Name":"duration","Value":"19","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9431660,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Value":81,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Value":53455371,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":777,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.shuffle.write.writeTime","Value":4719440,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1784265095239,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- == Current Plan ==\n   CollectLimit (26)\n   +- Project (25)\n      +- SortMergeJoin LeftOuter (24)\n         :- Sort (14)\n         :  +- LocalLimit (13)\n         :     +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :        :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :        :  +- Exchange (2)\n         :        :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :        +- BroadcastQueryStage (11)\n         :           +- BroadcastExchange (10)\n         :              +- * HashAggregate (9)\n         :                 +- AQEShuffleRead (8)\n         :                    +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                       +- Exchange (6)\n         :                          +- * HashAggregate (5)\n         :                             +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (23)\n            +- HashAggregate (22)\n               +- Exchange (21)\n                  +- HashAggregate (20)\n                     +- HashAggregate (19)\n                        +- ShuffleQueryStage (18)\n                           +- Exchange (17)\n                              +- * HashAggregate (16)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (42)\n   +- Project (41)\n      +- SortMergeJoin LeftOuter (40)\n         :- LocalLimit (33)\n         :  +- SortMergeJoin LeftAnti (32)\n         :     :- Sort (28)\n         :     :  +- Exchange (27)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (31)\n         :        +- HashAggregate (30)\n         :           +- Exchange (29)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (39)\n            +- HashAggregate (38)\n               +- Exchange (37)\n                  +- HashAggregate (36)\n                     +- HashAggregate (35)\n                        +- Exchange (34)\n                           +- HashAggregate (16)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(15) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(16) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(17) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(18) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(19) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]\n\n(22) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(23) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(24) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(25) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(26) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(27) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(28) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(30) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(31) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(33) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(34) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(35) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(36) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(37) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(38) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(39) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(40) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(41) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(42) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(43) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1140,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"spill size","accumulatorId":1139,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":983,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":984,"metricType":"timing"},{"name":"peak memory","accumulatorId":982,"metricType":"size"},{"name":"number of output rows","accumulatorId":981,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":986,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":985,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":977,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":978,"metricType":"timing"},{"name":"peak memory","accumulatorId":976,"metricType":"size"},{"name":"number of output rows","accumulatorId":975,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":980,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":979,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":973,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":967,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":974,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":968,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":965,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":962,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":971,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":964,"metricType":"sum"},{"name":"records read","accumulatorId":961,"metricType":"sum"},{"name":"local bytes read","accumulatorId":959,"metricType":"size"},{"name":"fetch wait time","accumulatorId":960,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":957,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":963,"metricType":"sum"},{"name":"local blocks read","accumulatorId":956,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":966,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":955,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":969,"metricType":"size"},{"name":"number of partitions","accumulatorId":954,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":970,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":958,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":972,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":949,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":950,"metricType":"timing"},{"name":"peak memory","accumulatorId":948,"metricType":"size"},{"name":"number of output rows","accumulatorId":947,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":952,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":951,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1156,"metricType":"timing"},{"name":"peak memory","accumulatorId":1157,"metricType":"size"},{"name":"spill size","accumulatorId":1158,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1135,"metricType":"sum"},{"name":"spill size","accumulatorId":1136,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1133,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1127,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1134,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1128,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1125,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1122,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1131,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1124,"metricType":"sum"},{"name":"records read","accumulatorId":1121,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1119,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1120,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1117,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1123,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1116,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1126,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1115,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1129,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1130,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1118,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1132,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":14,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095162,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095265,"Failed":false,"Killed":false,"Accumulables":[{"ID":855,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":856,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":987,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"shuffle bytes written","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1008,"Name":"shuffle write time","Update":"4426230","Value":"4426230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"duration","Update":"29","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1080,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8046722,"Value":8046722,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.executorRunTime","Update":81,"Value":81,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.executorCpuTime","Update":57495167,"Value":57495167,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1089,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.write.writeTime","Update":4426230,"Value":4426230,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":8046722,"Executor Run Time":81,"Executor CPU Time":57495167,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":3290,"Shuffle Write Time":4426230,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095156,"Completion Time":1784265095269,"Accumulables":[{"ID":855,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":856,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":987,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"shuffle bytes written","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1008,"Name":"shuffle write time","Value":"4426230","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"duration","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1080,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8046722,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.executorRunTime","Value":81,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.executorCpuTime","Value":57495167,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1089,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.write.writeTime","Value":4426230,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":8,"Completion Time":1784265095270,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1153,1],[1155,1],[1154,792]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Current Plan ==\n   CollectLimit (27)\n   +- Project (26)\n      +- BroadcastHashJoin LeftOuter BuildRight (25)\n         :- LocalLimit (13)\n         :  +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :     :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :  +- Exchange (2)\n         :     :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (11)\n         :        +- BroadcastExchange (10)\n         :           +- * HashAggregate (9)\n         :              +- AQEShuffleRead (8)\n         :                 +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (6)\n         :                       +- * HashAggregate (5)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastExchange (24)\n            +- HashAggregate (23)\n               +- ShuffleQueryStage (22)\n                  +- Exchange (21)\n                     +- * HashAggregate (20)\n                        +- * HashAggregate (19)\n                           +- AQEShuffleRead (18)\n                              +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                 +- Exchange (16)\n                                    +- * HashAggregate (15)\n                                       +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   CollectLimit (43)\n   +- Project (42)\n      +- SortMergeJoin LeftOuter (41)\n         :- LocalLimit (34)\n         :  +- SortMergeJoin LeftAnti (33)\n         :     :- Sort (29)\n         :     :  +- Exchange (28)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (32)\n         :        +- HashAggregate (31)\n         :           +- Exchange (30)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (40)\n            +- HashAggregate (39)\n               +- Exchange (38)\n                  +- HashAggregate (37)\n                     +- HashAggregate (36)\n                        +- Exchange (35)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(22) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(23) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=607]\n\n(25) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(26) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(27) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(28) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(29) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(30) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(31) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(32) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(33) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(34) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(35) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(36) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(37) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(38) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(39) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(40) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(42) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(43) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(44) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1180,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=607]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1191,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1185,"metricType":"timing"},{"name":"time to build","accumulatorId":1184,"metricType":"timing"},{"name":"time to collect","accumulatorId":1183,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1182,"metricType":"sum"},{"name":"data size","accumulatorId":1181,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1179,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1177,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1171,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1178,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1172,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1169,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1166,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1175,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1168,"metricType":"sum"},{"name":"records read","accumulatorId":1165,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1163,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1164,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1161,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1167,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1160,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1170,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1159,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1173,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1174,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1162,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1176,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":9,"Submission Time":1784265095286,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"72\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[15,16],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e829c027-dd2d-43b8-8f30-17206ec487ad)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"72\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095288,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e829c027-dd2d-43b8-8f30-17206ec487ad)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":16,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095294,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1227,1],[1229,1],[1228,3542]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1193,200]]}
{"Event":"SparkListenerJobStart","Job ID":10,"Submission Time":1784265095328,"Stage Infos":[{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[17,18],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095331,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095339,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":16,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095294,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095347,"Failed":false,"Killed":false,"Accumulables":[{"ID":924,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":927,"Name":"local bytes read","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":928,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":929,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1146,"Name":"duration","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1148,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1150,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8070102,"Value":8070102,"Internal":true,"Count Failed Values":true},{"ID":1232,"Name":"internal.metrics.executorRunTime","Update":32,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":1233,"Name":"internal.metrics.executorCpuTime","Update":30417882,"Value":30417882,"Internal":true,"Count Failed Values":true},{"ID":1234,"Name":"internal.metrics.resultSize","Update":4986,"Value":4986,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1244,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1245,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1247,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1248,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1249,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1257,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":8070102,"Executor Run Time":32,"Executor CPU Time":30417882,"Peak Execution Memory":262144,"Result Size":4986,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":777,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"72\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095288,"Completion Time":1784265095350,"Accumulables":[{"ID":924,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":927,"Name":"local bytes read","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":928,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":929,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1146,"Name":"duration","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1148,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1150,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8070102,"Internal":true,"Count Failed Values":true},{"ID":1232,"Name":"internal.metrics.executorRunTime","Value":32,"Internal":true,"Count Failed Values":true},{"ID":1233,"Name":"internal.metrics.executorCpuTime","Value":30417882,"Internal":true,"Count Failed Values":true},{"ID":1234,"Name":"internal.metrics.resultSize","Value":4986,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1244,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1245,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":777,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1247,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1248,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1249,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1257,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":9,"Completion Time":1784265095350,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1145,3],[1144,9],[1143,97],[1142,11],[1141,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095339,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095437,"Failed":false,"Killed":false,"Accumulables":[{"ID":990,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":993,"Name":"local bytes read","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1192,"Name":"data size","Update":"800","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1211,"Name":"shuffle bytes written","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1212,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1213,"Name":"shuffle write time","Update":"3073876","Value":"3073876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1214,"Name":"duration","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1215,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1216,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1218,"Name":"time in aggregation build","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1221,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1222,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1224,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1265,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":1266,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11363236,"Value":11363236,"Internal":true,"Count Failed Values":true},{"ID":1267,"Name":"internal.metrics.executorRunTime","Update":70,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":1268,"Name":"internal.metrics.executorCpuTime","Update":65469104,"Value":65469104,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1274,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.write.writeTime","Update":3073876,"Value":3073876,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":11363236,"Executor Run Time":70,"Executor CPU Time":65469104,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3290,"Total Records Read":45,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1503,"Shuffle Write Time":3073876,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095331,"Completion Time":1784265095439,"Accumulables":[{"ID":990,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":993,"Name":"local bytes read","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1192,"Name":"data size","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1211,"Name":"shuffle bytes written","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1212,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1213,"Name":"shuffle write time","Value":"3073876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1214,"Name":"duration","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1215,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1216,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1218,"Name":"time in aggregation build","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1221,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1222,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1224,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1265,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":1266,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11363236,"Internal":true,"Count Failed Values":true},{"ID":1267,"Name":"internal.metrics.executorRunTime","Value":70,"Internal":true,"Count Failed Values":true},{"ID":1268,"Name":"internal.metrics.executorCpuTime","Value":65469104,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1274,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.recordsRead","Value":45,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.write.writeTime","Value":3073876,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":10,"Completion Time":1784265095440,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (46)\n+- == Current Plan ==\n   CollectLimit (29)\n   +- Project (28)\n      +- BroadcastHashJoin LeftOuter BuildRight (27)\n         :- LocalLimit (13)\n         :  +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :     :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :  +- Exchange (2)\n         :     :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (11), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :        +- BroadcastExchange (10)\n         :           +- * HashAggregate (9)\n         :              +- AQEShuffleRead (8)\n         :                 +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (6)\n         :                       +- * HashAggregate (5)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastQueryStage (26)\n            +- BroadcastExchange (25)\n               +- * HashAggregate (24)\n                  +- AQEShuffleRead (23)\n                     +- ShuffleQueryStage (22), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (21)\n                           +- * HashAggregate (20)\n                              +- * HashAggregate (19)\n                                 +- AQEShuffleRead (18)\n                                    +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (16)\n                                          +- * HashAggregate (15)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- SortMergeJoin LeftOuter (43)\n         :- LocalLimit (36)\n         :  +- SortMergeJoin LeftAnti (35)\n         :     :- Sort (31)\n         :     :  +- Exchange (30)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (34)\n         :        +- HashAggregate (33)\n         :           +- Exchange (32)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (42)\n            +- HashAggregate (41)\n               +- Exchange (40)\n                  +- HashAggregate (39)\n                     +- HashAggregate (38)\n                        +- Exchange (37)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(22) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(23) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(24) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(25) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]\n\n(26) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(27) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(29) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(30) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(31) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(32) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(33) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(34) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(35) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(36) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(37) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(38) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(39) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(40) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(41) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(42) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(45) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(46) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1180,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1333,"metricType":"sum"},{"name":"partition data size","accumulatorId":1334,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1335,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1329,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1330,"metricType":"timing"},{"name":"peak memory","accumulatorId":1328,"metricType":"size"},{"name":"number of output rows","accumulatorId":1327,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1332,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1331,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1326,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1325,"metricType":"timing"},{"name":"time to build","accumulatorId":1324,"metricType":"timing"},{"name":"time to collect","accumulatorId":1323,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1322,"metricType":"sum"},{"name":"data size","accumulatorId":1321,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1320,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1318,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1312,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1319,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1313,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1310,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1307,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1316,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1309,"metricType":"sum"},{"name":"records read","accumulatorId":1306,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1304,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1305,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1302,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1308,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1301,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1311,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1300,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1314,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1315,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1303,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1317,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1333,1],[1335,1],[1334,1592]]}
{"Event":"SparkListenerJobStart","Job ID":11,"Submission Time":1784265095482,"Stage Infos":[{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"74\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[19],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[19,20,21],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId ca28d543-7e4c-4292-8079-91fe2bc1c7c8)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095486,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId ca28d543-7e4c-4292-8079-91fe2bc1c7c8)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":21,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095491,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":21,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095491,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095536,"Failed":false,"Killed":false,"Accumulables":[{"ID":1195,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1198,"Name":"local bytes read","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1199,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1200,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1331,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1336,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9252966,"Value":9252966,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.executorRunTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.executorCpuTime","Update":20435995,"Value":20435995,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.resultSize","Update":7089,"Value":7089,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1359,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1360,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1361,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1362,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1363,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":9252966,"Executor Run Time":22,"Executor CPU Time":20435995,"Peak Execution Memory":4456432,"Result Size":7089,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1503,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095486,"Completion Time":1784265095537,"Accumulables":[{"ID":1195,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1198,"Name":"local bytes read","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1199,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1200,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1331,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1336,"Name":"internal.metrics.executorDeserializeTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9252966,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.executorRunTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.executorCpuTime","Value":20435995,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.resultSize","Value":7089,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1359,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1360,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1361,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1362,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1363,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":11,"Completion Time":1784265095538,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1325,3],[1324,9],[1323,85],[1322,20],[1321,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (47)\n+- == Final Plan ==\n   CollectLimit (30)\n   +- * Project (29)\n      +- * BroadcastHashJoin LeftOuter BuildRight (28)\n         :- * LocalLimit (14)\n         :  +- * BroadcastHashJoin LeftAnti BuildRight (13)\n         :     :- AQEShuffleRead (4)\n         :     :  +- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :     +- Exchange (2)\n         :     :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (12), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :        +- BroadcastExchange (11)\n         :           +- * HashAggregate (10)\n         :              +- AQEShuffleRead (9)\n         :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (7)\n         :                       +- * HashAggregate (6)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- BroadcastQueryStage (27), Statistics(sizeInBytes=2.0 MiB, rowCount=20)\n            +- BroadcastExchange (26)\n               +- * HashAggregate (25)\n                  +- AQEShuffleRead (24)\n                     +- ShuffleQueryStage (23), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (22)\n                           +- * HashAggregate (21)\n                              +- * HashAggregate (20)\n                                 +- AQEShuffleRead (19)\n                                    +- ShuffleQueryStage (18), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (17)\n                                          +- * HashAggregate (16)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (46)\n   +- Project (45)\n      +- SortMergeJoin LeftOuter (44)\n         :- LocalLimit (37)\n         :  +- SortMergeJoin LeftAnti (36)\n         :     :- Sort (32)\n         :     :  +- Exchange (31)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (35)\n         :        +- HashAggregate (34)\n         :           +- Exchange (33)\n         :              +- HashAggregate (6)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (43)\n            +- HashAggregate (42)\n               +- Exchange (41)\n                  +- HashAggregate (40)\n                     +- HashAggregate (39)\n                        +- Exchange (38)\n                           +- HashAggregate (16)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: local\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(11) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(12) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(13) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(14) LocalLimit [codegen id : 7]\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(15) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(16) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(17) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(18) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(19) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(21) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(22) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(23) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(24) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(25) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(26) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]\n\n(27) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(28) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(29) Project [codegen id : 7]\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(30) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(31) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(32) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(33) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(34) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(35) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(37) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(38) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(39) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(40) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(41) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(42) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(43) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(44) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(45) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(46) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(47) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1394,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1393,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1333,"metricType":"sum"},{"name":"partition data size","accumulatorId":1334,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1335,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1329,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1330,"metricType":"timing"},{"name":"peak memory","accumulatorId":1328,"metricType":"size"},{"name":"number of output rows","accumulatorId":1327,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1332,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1331,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1326,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1325,"metricType":"timing"},{"name":"time to build","accumulatorId":1324,"metricType":"timing"},{"name":"time to collect","accumulatorId":1323,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1322,"metricType":"sum"},{"name":"data size","accumulatorId":1321,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1392,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1391,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1389,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1383,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1390,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1384,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1381,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1378,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1387,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1380,"metricType":"sum"},{"name":"records read","accumulatorId":1377,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1375,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1376,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1373,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1379,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1372,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1382,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1371,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1385,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1386,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1374,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1388,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1394,1]]}
{"Event":"SparkListenerJobStart","Job ID":12,"Submission Time":1784265095616,"Stage Infos":[{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[22,23],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095617,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":23,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095622,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":23,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095622,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095685,"Failed":false,"Killed":false,"Accumulables":[{"ID":892,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":895,"Name":"local bytes read","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":896,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":897,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"duration","Update":"27","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1392,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":1396,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6935447,"Value":6935447,"Internal":true,"Count Failed Values":true},{"ID":1397,"Name":"internal.metrics.executorRunTime","Update":43,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":1398,"Name":"internal.metrics.executorCpuTime","Update":26197519,"Value":26197519,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.resultSize","Update":4400,"Value":4400,"Internal":true,"Count Failed Values":true},{"ID":1404,"Name":"internal.metrics.peakExecutionMemory","Update":8390624,"Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":1406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1410,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1412,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":6935447,"Executor Run Time":43,"Executor CPU Time":26197519,"Peak Execution Memory":8390624,"Result Size":4400,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2299,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095617,"Completion Time":1784265095686,"Accumulables":[{"ID":892,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":895,"Name":"local bytes read","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":896,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":897,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"duration","Value":"27","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1392,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":1396,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6935447,"Internal":true,"Count Failed Values":true},{"ID":1397,"Name":"internal.metrics.executorRunTime","Value":43,"Internal":true,"Count Failed Values":true},{"ID":1398,"Name":"internal.metrics.executorCpuTime","Value":26197519,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.resultSize","Value":4400,"Internal":true,"Count Failed Values":true},{"ID":1404,"Name":"internal.metrics.peakExecutionMemory","Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":1406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1410,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1412,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":12,"Completion Time":1784265095686,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1784265095693,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"rootExecutionId":2,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (40)\n+- CollectLimit (39)\n   +- Project (38)\n      +- SortMergeJoin LeftOuter (37)\n         :- Sort (31)\n         :  +- Exchange (30)\n         :     +- LocalLimit (29)\n         :        +- Project (28)\n         :           +- SortMergeJoin LeftOuter (27)\n         :              :- Sort (18)\n         :              :  +- Exchange (17)\n         :              :     +- LocalLimit (16)\n         :              :        +- Project (15)\n         :              :           +- SortMergeJoin Inner (14)\n         :              :              :- Sort (10)\n         :              :              :  +- Exchange (9)\n         :              :              :     +- Project (8)\n         :              :              :        +- SortMergeJoin Inner (7)\n         :              :              :           :- Sort (3)\n         :              :              :           :  +- Exchange (2)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (6)\n         :              :              :              +- Exchange (5)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (4)\n         :              :              +- Sort (13)\n         :              :                 +- Exchange (12)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (11)\n         :              +- Sort (26)\n         :                 +- HashAggregate (25)\n         :                    +- Exchange (24)\n         :                       +- HashAggregate (23)\n         :                          +- HashAggregate (22)\n         :                             +- Exchange (21)\n         :                                +- HashAggregate (20)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (19)\n         +- Sort (36)\n            +- HashAggregate (35)\n               +- Exchange (34)\n                  +- HashAggregate (33)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (32)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(3) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(5) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(6) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(9) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(10) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(12) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(13) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(15) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(16) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(17) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(18) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(19) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(20) HashAggregate\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(21) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(22) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(25) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(26) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(27) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(29) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(30) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(31) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(32) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(33) HashAggregate\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(34) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(35) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(36) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(39) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(40) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1556,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1550,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1557,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1551,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1548,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1545,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1554,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1547,"metricType":"sum"},{"name":"records read","accumulatorId":1544,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1542,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1543,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1540,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1546,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1539,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1549,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1538,"metricType":"sum"},{"name":"data size","accumulatorId":1536,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1552,"metricType":"size"},{"name":"number of partitions","accumulatorId":1537,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1553,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1541,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1555,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1533,"metricType":"timing"},{"name":"peak memory","accumulatorId":1534,"metricType":"size"},{"name":"spill size","accumulatorId":1535,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1582,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1576,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1583,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1577,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1574,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1571,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1580,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1573,"metricType":"sum"},{"name":"records read","accumulatorId":1570,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1568,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1569,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1566,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1572,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1565,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1575,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1564,"metricType":"sum"},{"name":"data size","accumulatorId":1562,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1578,"metricType":"size"},{"name":"number of partitions","accumulatorId":1563,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1579,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1567,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1581,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1559,"metricType":"timing"},{"name":"peak memory","accumulatorId":1560,"metricType":"size"},{"name":"spill size","accumulatorId":1561,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1531,"metricType":"sum"},{"name":"spill size","accumulatorId":1532,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1529,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1523,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1530,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1524,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1521,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1518,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1527,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1520,"metricType":"sum"},{"name":"records read","accumulatorId":1517,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1515,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1516,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1513,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1519,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1512,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1522,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1511,"metricType":"sum"},{"name":"data size","accumulatorId":1509,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1525,"metricType":"size"},{"name":"number of partitions","accumulatorId":1510,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1526,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1514,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1528,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1506,"metricType":"timing"},{"name":"peak memory","accumulatorId":1507,"metricType":"size"},{"name":"spill size","accumulatorId":1508,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1608,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1602,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1609,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1603,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1600,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1597,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1606,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1599,"metricType":"sum"},{"name":"records read","accumulatorId":1596,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1594,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1595,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1592,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1598,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1591,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1601,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1590,"metricType":"sum"},{"name":"data size","accumulatorId":1588,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1604,"metricType":"size"},{"name":"number of partitions","accumulatorId":1589,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1605,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1593,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1607,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1585,"metricType":"timing"},{"name":"peak memory","accumulatorId":1586,"metricType":"size"},{"name":"spill size","accumulatorId":1587,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1504,"metricType":"sum"},{"name":"spill size","accumulatorId":1505,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1502,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1496,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1503,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1497,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1494,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1491,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1500,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1493,"metricType":"sum"},{"name":"records read","accumulatorId":1490,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1488,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1489,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1486,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1492,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1485,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1495,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1484,"metricType":"sum"},{"name":"data size","accumulatorId":1482,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1498,"metricType":"size"},{"name":"number of partitions","accumulatorId":1483,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1499,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1487,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1501,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1479,"metricType":"timing"},{"name":"peak memory","accumulatorId":1480,"metricType":"size"},{"name":"spill size","accumulatorId":1481,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1674,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1668,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1675,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1669,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1666,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1663,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1672,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1665,"metricType":"sum"},{"name":"records read","accumulatorId":1662,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1660,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1661,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1658,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1664,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1657,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1667,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1656,"metricType":"sum"},{"name":"data size","accumulatorId":1654,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1670,"metricType":"size"},{"name":"number of partitions","accumulatorId":1655,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1671,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1659,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1673,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1650,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1651,"metricType":"timing"},{"name":"peak memory","accumulatorId":1649,"metricType":"size"},{"name":"number of output rows","accumulatorId":1648,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1653,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1652,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1644,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1645,"metricType":"timing"},{"name":"peak memory","accumulatorId":1643,"metricType":"size"},{"name":"number of output rows","accumulatorId":1642,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1647,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1646,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1640,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1634,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1641,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1635,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1632,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1629,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1638,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1631,"metricType":"sum"},{"name":"records read","accumulatorId":1628,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1626,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1627,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1624,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1630,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1623,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1633,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1622,"metricType":"sum"},{"name":"data size","accumulatorId":1620,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1636,"metricType":"size"},{"name":"number of partitions","accumulatorId":1621,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1637,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1625,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1639,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1616,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1617,"metricType":"timing"},{"name":"peak memory","accumulatorId":1615,"metricType":"size"},{"name":"number of output rows","accumulatorId":1614,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1619,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1618,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1611,"metricType":"timing"},{"name":"peak memory","accumulatorId":1612,"metricType":"size"},{"name":"spill size","accumulatorId":1613,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1477,"metricType":"sum"},{"name":"spill size","accumulatorId":1478,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1475,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1469,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1476,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1470,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1467,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1464,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1473,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1466,"metricType":"sum"},{"name":"records read","accumulatorId":1463,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1461,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1462,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1459,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1465,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1458,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1468,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1457,"metricType":"sum"},{"name":"data size","accumulatorId":1455,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1471,"metricType":"size"},{"name":"number of partitions","accumulatorId":1456,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1472,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1460,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1474,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1452,"metricType":"timing"},{"name":"peak memory","accumulatorId":1453,"metricType":"size"},{"name":"spill size","accumulatorId":1454,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1712,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1706,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1713,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1707,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1704,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1701,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1710,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1703,"metricType":"sum"},{"name":"records read","accumulatorId":1700,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1698,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1699,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1696,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1702,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1695,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1705,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1694,"metricType":"sum"},{"name":"data size","accumulatorId":1692,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1708,"metricType":"size"},{"name":"number of partitions","accumulatorId":1693,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1709,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1697,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1711,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1688,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1689,"metricType":"timing"},{"name":"peak memory","accumulatorId":1687,"metricType":"size"},{"name":"number of output rows","accumulatorId":1686,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1691,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1690,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1683,"metricType":"timing"},{"name":"peak memory","accumulatorId":1684,"metricType":"size"},{"name":"spill size","accumulatorId":1685,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1450,"metricType":"sum"},{"name":"spill size","accumulatorId":1451,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1448,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1442,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1449,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1443,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1440,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1437,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1446,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1439,"metricType":"sum"},{"name":"records read","accumulatorId":1436,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1434,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1435,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1432,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1438,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1431,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1441,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1430,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1444,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1445,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1433,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1447,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784265095767,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (77)\n+- == Current Plan ==\n   CollectLimit (44)\n   +- Project (43)\n      +- SortMergeJoin LeftOuter (42)\n         :- Sort (35)\n         :  +- Exchange (34)\n         :     +- LocalLimit (33)\n         :        +- Project (32)\n         :           +- SortMergeJoin LeftOuter (31)\n         :              :- Sort (21)\n         :              :  +- Exchange (20)\n         :              :     +- LocalLimit (19)\n         :              :        +- Project (18)\n         :              :           +- SortMergeJoin Inner (17)\n         :              :              :- Sort (12)\n         :              :              :  +- Exchange (11)\n         :              :              :     +- Project (10)\n         :              :              :        +- SortMergeJoin Inner (9)\n         :              :              :           :- Sort (4)\n         :              :              :           :  +- ShuffleQueryStage (3)\n         :              :              :           :     +- Exchange (2)\n         :              :              :           :        +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (8)\n         :              :              :              +- ShuffleQueryStage (7)\n         :              :              :                 +- Exchange (6)\n         :              :              :                    +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :              :              +- Sort (16)\n         :              :                 +- ShuffleQueryStage (15)\n         :              :                    +- Exchange (14)\n         :              :                       +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :              +- Sort (30)\n         :                 +- HashAggregate (29)\n         :                    +- Exchange (28)\n         :                       +- HashAggregate (27)\n         :                          +- HashAggregate (26)\n         :                             +- ShuffleQueryStage (25)\n         :                                +- Exchange (24)\n         :                                   +- * HashAggregate (23)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (41)\n            +- HashAggregate (40)\n               +- ShuffleQueryStage (39)\n                  +- Exchange (38)\n                     +- * HashAggregate (37)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n+- == Initial Plan ==\n   CollectLimit (76)\n   +- Project (75)\n      +- SortMergeJoin LeftOuter (74)\n         :- Sort (70)\n         :  +- Exchange (69)\n         :     +- LocalLimit (68)\n         :        +- Project (67)\n         :           +- SortMergeJoin LeftOuter (66)\n         :              :- Sort (59)\n         :              :  +- Exchange (58)\n         :              :     +- LocalLimit (57)\n         :              :        +- Project (56)\n         :              :           +- SortMergeJoin Inner (55)\n         :              :              :- Sort (52)\n         :              :              :  +- Exchange (51)\n         :              :              :     +- Project (50)\n         :              :              :        +- SortMergeJoin Inner (49)\n         :              :              :           :- Sort (46)\n         :              :              :           :  +- Exchange (45)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (48)\n         :              :              :              +- Exchange (47)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :              :              +- Sort (54)\n         :              :                 +- Exchange (53)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :              +- Sort (65)\n         :                 +- HashAggregate (64)\n         :                    +- Exchange (63)\n         :                       +- HashAggregate (62)\n         :                          +- HashAggregate (61)\n         :                             +- Exchange (60)\n         :                                +- HashAggregate (23)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (73)\n            +- HashAggregate (72)\n               +- Exchange (71)\n                  +- HashAggregate (37)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(6) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(7) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(8) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(11) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=910]\n\n(12) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(14) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(15) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(16) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(18) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(19) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(20) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=922]\n\n(21) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(22) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(23) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(25) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(26) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(29) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(30) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(32) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(33) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(34) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=938]\n\n(35) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(36) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(38) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(40) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(42) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(43) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(44) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(45) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(46) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(47) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(48) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(49) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(50) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(51) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(52) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(53) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(54) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(55) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(56) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(57) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(58) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(59) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(60) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(61) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(63) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(64) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(65) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(67) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(68) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(69) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(70) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(71) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(72) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(73) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(74) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(75) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(76) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(77) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=938]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=922]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=910]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1824,"metricType":"timing"},{"name":"peak memory","accumulatorId":1825,"metricType":"size"},{"name":"spill size","accumulatorId":1826,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1850,"metricType":"timing"},{"name":"peak memory","accumulatorId":1851,"metricType":"size"},{"name":"spill size","accumulatorId":1852,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1822,"metricType":"sum"},{"name":"spill size","accumulatorId":1823,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1820,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1814,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1821,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1815,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1812,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1809,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1818,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1811,"metricType":"sum"},{"name":"records read","accumulatorId":1808,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1806,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1807,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1804,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1810,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1803,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1813,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1802,"metricType":"sum"},{"name":"data size","accumulatorId":1800,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1816,"metricType":"size"},{"name":"number of partitions","accumulatorId":1801,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1817,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1805,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1819,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1797,"metricType":"timing"},{"name":"peak memory","accumulatorId":1798,"metricType":"size"},{"name":"spill size","accumulatorId":1799,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1876,"metricType":"timing"},{"name":"peak memory","accumulatorId":1877,"metricType":"size"},{"name":"spill size","accumulatorId":1878,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1795,"metricType":"sum"},{"name":"spill size","accumulatorId":1796,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1793,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1787,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1794,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1788,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1785,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1782,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1791,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1784,"metricType":"sum"},{"name":"records read","accumulatorId":1781,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1779,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1780,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1777,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1783,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1776,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1786,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1775,"metricType":"sum"},{"name":"data size","accumulatorId":1773,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1789,"metricType":"size"},{"name":"number of partitions","accumulatorId":1774,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1790,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1778,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1792,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1770,"metricType":"timing"},{"name":"peak memory","accumulatorId":1771,"metricType":"size"},{"name":"spill size","accumulatorId":1772,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1902,"metricType":"timing"},{"name":"peak memory","accumulatorId":1903,"metricType":"size"},{"name":"spill size","accumulatorId":1904,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1768,"metricType":"sum"},{"name":"spill size","accumulatorId":1769,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1766,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1760,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1767,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1761,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1758,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1755,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1764,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1757,"metricType":"sum"},{"name":"records read","accumulatorId":1754,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1752,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1753,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1750,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1756,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1749,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1759,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1748,"metricType":"sum"},{"name":"data size","accumulatorId":1746,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1762,"metricType":"size"},{"name":"number of partitions","accumulatorId":1747,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1763,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1751,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1765,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1743,"metricType":"timing"},{"name":"peak memory","accumulatorId":1744,"metricType":"size"},{"name":"spill size","accumulatorId":1745,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1973,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1974,"metricType":"timing"},{"name":"peak memory","accumulatorId":1972,"metricType":"size"},{"name":"number of output rows","accumulatorId":1971,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1976,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1975,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1968,"metricType":"timing"},{"name":"peak memory","accumulatorId":1969,"metricType":"size"},{"name":"spill size","accumulatorId":1970,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1741,"metricType":"sum"},{"name":"spill size","accumulatorId":1742,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1739,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1733,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1740,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1734,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1731,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1728,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1737,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1730,"metricType":"sum"},{"name":"records read","accumulatorId":1727,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1725,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1726,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1723,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1729,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1722,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1732,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1721,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1735,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1736,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1724,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1738,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1828,200]]}
{"Event":"SparkListenerJobStart","Job ID":13,"Submission Time":1784265095789,"Stage Infos":[{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[24],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095789,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1854,200]]}
{"Event":"SparkListenerJobStart","Job ID":14,"Submission Time":1784265095794,"Stage Infos":[{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[25],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095795,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1880,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":24,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095794,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":15,"Submission Time":1784265095800,"Stage Infos":[{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[26],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095800,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":25,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095799,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":26,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095809,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1946,200]]}
{"Event":"SparkListenerJobStart","Job ID":16,"Submission Time":1784265095827,"Stage Infos":[{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[27],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095831,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":27,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095839,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1978,200]]}
{"Event":"SparkListenerJobStart","Job ID":17,"Submission Time":1784265095861,"Stage Infos":[{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[28],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095866,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":28,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095876,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":24,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095794,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095932,"Failed":false,"Killed":false,"Accumulables":[{"ID":1558,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1827,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1846,"Name":"shuffle bytes written","Update":"1849","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1847,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1848,"Name":"shuffle write time","Update":"6416761","Value":"6416761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1849,"Name":"duration","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":2001,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8563792,"Value":8563792,"Internal":true,"Count Failed Values":true},{"ID":2002,"Name":"internal.metrics.executorRunTime","Update":82,"Value":82,"Internal":true,"Count Failed Values":true},{"ID":2003,"Name":"internal.metrics.executorCpuTime","Update":30895828,"Value":30895828,"Internal":true,"Count Failed Values":true},{"ID":2004,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2028,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1849,"Value":1849,"Internal":true,"Count Failed Values":true},{"ID":2029,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2030,"Name":"internal.metrics.shuffle.write.writeTime","Update":6416761,"Value":6416761,"Internal":true,"Count Failed Values":true},{"ID":2032,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":8563792,"Executor Run Time":82,"Executor CPU Time":30895828,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1849,"Shuffle Write Time":6416761,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095789,"Completion Time":1784265095934,"Accumulables":[{"ID":1558,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1827,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1846,"Name":"shuffle bytes written","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1847,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1848,"Name":"shuffle write time","Value":"6416761","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1849,"Name":"duration","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"internal.metrics.executorDeserializeTime","Value":44,"Internal":true,"Count Failed Values":true},{"ID":2001,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8563792,"Internal":true,"Count Failed Values":true},{"ID":2002,"Name":"internal.metrics.executorRunTime","Value":82,"Internal":true,"Count Failed Values":true},{"ID":2003,"Name":"internal.metrics.executorCpuTime","Value":30895828,"Internal":true,"Count Failed Values":true},{"ID":2004,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2028,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1849,"Internal":true,"Count Failed Values":true},{"ID":2029,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":2030,"Name":"internal.metrics.shuffle.write.writeTime","Value":6416761,"Internal":true,"Count Failed Values":true},{"ID":2032,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":13,"Completion Time":1784265095939,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":26,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095809,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095937,"Failed":false,"Killed":false,"Accumulables":[{"ID":1610,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1879,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1898,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1899,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1900,"Name":"shuffle write time","Update":"5461660","Value":"5461660","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1901,"Name":"duration","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2070,"Name":"internal.metrics.executorDeserializeTime","Update":22,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7272993,"Value":7272993,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.executorRunTime","Update":89,"Value":89,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.executorCpuTime","Update":27232360,"Value":27232360,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.write.writeTime","Update":5461660,"Value":5461660,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":22,"Executor Deserialize CPU Time":7272993,"Executor Run Time":89,"Executor CPU Time":27232360,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":5461660,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095800,"Completion Time":1784265095941,"Accumulables":[{"ID":1610,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1879,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1898,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1899,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1900,"Name":"shuffle write time","Value":"5461660","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1901,"Name":"duration","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2070,"Name":"internal.metrics.executorDeserializeTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7272993,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.executorRunTime","Value":89,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.executorCpuTime","Value":27232360,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.write.writeTime","Value":5461660,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":15,"Completion Time":1784265095943,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":25,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095799,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265095939,"Failed":false,"Killed":false,"Accumulables":[{"ID":1584,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"data size","Update":"960","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"shuffle bytes written","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1873,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1874,"Name":"shuffle write time","Update":"5525958","Value":"5525958","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1875,"Name":"duration","Update":"22","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2035,"Name":"internal.metrics.executorDeserializeTime","Update":37,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":2036,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7196623,"Value":7196623,"Internal":true,"Count Failed Values":true},{"ID":2037,"Name":"internal.metrics.executorRunTime","Update":94,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":2038,"Name":"internal.metrics.executorCpuTime","Update":21969496,"Value":21969496,"Internal":true,"Count Failed Values":true},{"ID":2039,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1650,"Value":1650,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2065,"Name":"internal.metrics.shuffle.write.writeTime","Update":5525958,"Value":5525958,"Internal":true,"Count Failed Values":true},{"ID":2067,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":37,"Executor Deserialize CPU Time":7196623,"Executor Run Time":94,"Executor CPU Time":21969496,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1650,"Shuffle Write Time":5525958,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095795,"Completion Time":1784265095945,"Accumulables":[{"ID":1584,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"data size","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"shuffle bytes written","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1873,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1874,"Name":"shuffle write time","Value":"5525958","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1875,"Name":"duration","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2035,"Name":"internal.metrics.executorDeserializeTime","Value":37,"Internal":true,"Count Failed Values":true},{"ID":2036,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7196623,"Internal":true,"Count Failed Values":true},{"ID":2037,"Name":"internal.metrics.executorRunTime","Value":94,"Internal":true,"Count Failed Values":true},{"ID":2038,"Name":"internal.metrics.executorCpuTime","Value":21969496,"Internal":true,"Count Failed Values":true},{"ID":2039,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1650,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":2065,"Name":"internal.metrics.shuffle.write.writeTime","Value":5525958,"Internal":true,"Count Failed Values":true},{"ID":2067,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":14,"Completion Time":1784265095947,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- SortMergeJoin LeftOuter (43)\n         :- Sort (36)\n         :  +- Exchange (35)\n         :     +- LocalLimit (34)\n         :        +- Project (33)\n         :           +- SortMergeJoin LeftOuter (32)\n         :              :- Sort (22)\n         :              :  +- Exchange (21)\n         :              :     +- LocalLimit (20)\n         :              :        +- Project (19)\n         :              :           +- BroadcastHashJoin Inner BuildLeft (18)\n         :              :              :- BroadcastQueryStage (14)\n         :              :              :  +- BroadcastExchange (13)\n         :              :              :     +- * Project (12)\n         :              :              :        +- * SortMergeJoin Inner (11)\n         :              :              :           :- * Sort (5)\n         :              :              :           :  +- AQEShuffleRead (4)\n         :              :              :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :              :              :           :        +- Exchange (2)\n         :              :              :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- * Sort (10)\n         :              :              :              +- AQEShuffleRead (9)\n         :              :              :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :              :              :                    +- Exchange (7)\n         :              :              :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :              :                 +- Exchange (16)\n         :              :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (31)\n         :                 +- HashAggregate (30)\n         :                    +- Exchange (29)\n         :                       +- HashAggregate (28)\n         :                          +- HashAggregate (27)\n         :                             +- ShuffleQueryStage (26)\n         :                                +- Exchange (25)\n         :                                   +- * HashAggregate (24)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (42)\n            +- HashAggregate (41)\n               +- ShuffleQueryStage (40)\n                  +- Exchange (39)\n                     +- * HashAggregate (38)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (24)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (38)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1201]\n\n(22) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(23) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(24) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(25) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(26) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1206]\n\n(36) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(37) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(38) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(39) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(40) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(41) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(42) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1206]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1201]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2249,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2247,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2241,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2248,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2242,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2239,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2236,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2245,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2238,"metricType":"sum"},{"name":"records read","accumulatorId":2235,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2233,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2234,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2231,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2237,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2230,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2240,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2229,"metricType":"sum"},{"name":"data size","accumulatorId":2227,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2243,"metricType":"size"},{"name":"number of partitions","accumulatorId":2228,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2244,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2232,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2246,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2224,"metricType":"timing"},{"name":"peak memory","accumulatorId":2225,"metricType":"size"},{"name":"spill size","accumulatorId":2226,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2272,"metricType":"timing"},{"name":"peak memory","accumulatorId":2273,"metricType":"size"},{"name":"spill size","accumulatorId":2274,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2222,"metricType":"sum"},{"name":"spill size","accumulatorId":2223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2220,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2214,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2221,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2215,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2212,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2209,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2218,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2211,"metricType":"sum"},{"name":"records read","accumulatorId":2208,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2206,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2207,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2204,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2210,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2203,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2213,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2202,"metricType":"sum"},{"name":"data size","accumulatorId":2200,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2216,"metricType":"size"},{"name":"number of partitions","accumulatorId":2201,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2217,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2205,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2219,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2197,"metricType":"timing"},{"name":"peak memory","accumulatorId":2198,"metricType":"size"},{"name":"spill size","accumulatorId":2199,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1973,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1974,"metricType":"timing"},{"name":"peak memory","accumulatorId":1972,"metricType":"size"},{"name":"number of output rows","accumulatorId":1971,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1976,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1975,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2275,"metricType":"timing"},{"name":"peak memory","accumulatorId":2276,"metricType":"size"},{"name":"spill size","accumulatorId":2277,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2195,"metricType":"sum"},{"name":"spill size","accumulatorId":2196,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2193,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2187,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2194,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2188,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2185,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2182,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2191,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2184,"metricType":"sum"},{"name":"records read","accumulatorId":2181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2177,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2183,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2176,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2186,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2175,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2189,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2190,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2192,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":28,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095876,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096023,"Failed":false,"Killed":false,"Accumulables":[{"ID":1714,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1715,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1717,"Name":"time in aggregation build","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1720,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1977,"Name":"data size","Update":"200","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"shuffle bytes written","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"shuffle write time","Update":"8756648","Value":"8756648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"duration","Update":"25","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2140,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9368771,"Value":9368771,"Internal":true,"Count Failed Values":true},{"ID":2142,"Name":"internal.metrics.executorRunTime","Update":117,"Value":117,"Internal":true,"Count Failed Values":true},{"ID":2143,"Name":"internal.metrics.executorCpuTime","Update":53629531,"Value":53629531,"Internal":true,"Count Failed Values":true},{"ID":2144,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2149,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2168,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":2169,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2170,"Name":"internal.metrics.shuffle.write.writeTime","Update":8756648,"Value":8756648,"Internal":true,"Count Failed Values":true},{"ID":2172,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":9368771,"Executor Run Time":117,"Executor CPU Time":53629531,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":370,"Shuffle Write Time":8756648,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095866,"Completion Time":1784265096024,"Accumulables":[{"ID":1714,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1715,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1717,"Name":"time in aggregation build","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1720,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1977,"Name":"data size","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"shuffle bytes written","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"shuffle write time","Value":"8756648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"duration","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2140,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9368771,"Internal":true,"Count Failed Values":true},{"ID":2142,"Name":"internal.metrics.executorRunTime","Value":117,"Internal":true,"Count Failed Values":true},{"ID":2143,"Name":"internal.metrics.executorCpuTime","Value":53629531,"Internal":true,"Count Failed Values":true},{"ID":2144,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2149,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2168,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":370,"Internal":true,"Count Failed Values":true},{"ID":2169,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2170,"Name":"internal.metrics.shuffle.write.writeTime","Value":8756648,"Internal":true,"Count Failed Values":true},{"ID":2172,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":17,"Completion Time":1784265096025,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":27,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265095839,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096039,"Failed":false,"Killed":false,"Accumulables":[{"ID":1676,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1677,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1679,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1945,"Name":"data size","Update":"4320","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle bytes written","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"shuffle records written","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1966,"Name":"shuffle write time","Update":"6721084","Value":"6721084","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1967,"Name":"duration","Update":"47","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2105,"Name":"internal.metrics.executorDeserializeTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11243045,"Value":11243045,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.executorRunTime","Update":147,"Value":147,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.executorCpuTime","Update":87502813,"Value":87502813,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2114,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2133,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2134,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2135,"Name":"internal.metrics.shuffle.write.writeTime","Update":6721084,"Value":6721084,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":33,"Executor Deserialize CPU Time":11243045,"Executor Run Time":147,"Executor CPU Time":87502813,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":4311,"Shuffle Write Time":6721084,"Shuffle Records Written":54},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265095831,"Completion Time":1784265096040,"Accumulables":[{"ID":1676,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1677,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1679,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1945,"Name":"data size","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle bytes written","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"shuffle records written","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1966,"Name":"shuffle write time","Value":"6721084","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1967,"Name":"duration","Value":"47","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2105,"Name":"internal.metrics.executorDeserializeTime","Value":33,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11243045,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.executorRunTime","Value":147,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.executorCpuTime","Value":87502813,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2114,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2133,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2134,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":54,"Internal":true,"Count Failed Values":true},{"ID":2135,"Name":"internal.metrics.shuffle.write.writeTime","Value":6721084,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":16,"Completion Time":1784265096042,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2262,1],[2264,1],[2263,1984]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- BroadcastHashJoin LeftOuter BuildRight (43)\n         :- LocalLimit (34)\n         :  +- Project (33)\n         :     +- SortMergeJoin LeftOuter (32)\n         :        :- Sort (22)\n         :        :  +- Exchange (21)\n         :        :     +- LocalLimit (20)\n         :        :        +- Project (19)\n         :        :           +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :              :- BroadcastQueryStage (14)\n         :        :              :  +- BroadcastExchange (13)\n         :        :              :     +- * Project (12)\n         :        :              :        +- * SortMergeJoin Inner (11)\n         :        :              :           :- * Sort (5)\n         :        :              :           :  +- AQEShuffleRead (4)\n         :        :              :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :              :           :        +- Exchange (2)\n         :        :              :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :              :           +- * Sort (10)\n         :        :              :              +- AQEShuffleRead (9)\n         :        :              :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :              :                    +- Exchange (7)\n         :        :              :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :              +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :                 +- Exchange (16)\n         :        :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- Sort (31)\n         :           +- HashAggregate (30)\n         :              +- Exchange (29)\n         :                 +- HashAggregate (28)\n         :                    +- HashAggregate (27)\n         :                       +- ShuffleQueryStage (26)\n         :                          +- Exchange (25)\n         :                             +- * HashAggregate (24)\n         :                                +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- BroadcastQueryStage (42)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (24)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1245]\n\n(22) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(23) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(24) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(25) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(26) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]\n\n(42) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(43) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1245]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2326,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2324,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2325,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2319,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2316,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2313,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2322,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2315,"metricType":"sum"},{"name":"records read","accumulatorId":2312,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2310,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2311,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2308,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2314,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2307,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2317,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2306,"metricType":"sum"},{"name":"data size","accumulatorId":2304,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2320,"metricType":"size"},{"name":"number of partitions","accumulatorId":2305,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2321,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2309,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2323,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2301,"metricType":"timing"},{"name":"peak memory","accumulatorId":2302,"metricType":"size"},{"name":"spill size","accumulatorId":2303,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2327,"metricType":"timing"},{"name":"peak memory","accumulatorId":2328,"metricType":"size"},{"name":"spill size","accumulatorId":2329,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2299,"metricType":"sum"},{"name":"spill size","accumulatorId":2300,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2342,"metricType":"sum"},{"name":"partition data size","accumulatorId":2343,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2338,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2339,"metricType":"timing"},{"name":"peak memory","accumulatorId":2337,"metricType":"size"},{"name":"number of output rows","accumulatorId":2336,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2341,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2340,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2335,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2334,"metricType":"timing"},{"name":"time to build","accumulatorId":2333,"metricType":"timing"},{"name":"time to collect","accumulatorId":2332,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2331,"metricType":"sum"},{"name":"data size","accumulatorId":2330,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2298,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2296,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2290,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2297,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2291,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2285,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2294,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"records read","accumulatorId":2284,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2282,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2283,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2280,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2286,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2279,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2289,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2278,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2292,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2293,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2281,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2295,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2269,1],[2271,1],[2270,1744]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2342,1],[2344,1],[2343,392]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- BroadcastHashJoin LeftOuter BuildRight (43)\n         :- LocalLimit (34)\n         :  +- Project (33)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (32)\n         :        :- LocalLimit (20)\n         :        :  +- Project (19)\n         :        :     +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :        :- BroadcastQueryStage (14)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :           +- Exchange (16)\n         :        :              +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastExchange (31)\n         :           +- HashAggregate (30)\n         :              +- ShuffleQueryStage (29)\n         :                 +- Exchange (28)\n         :                    +- * HashAggregate (27)\n         :                       +- * HashAggregate (26)\n         :                          +- AQEShuffleRead (25)\n         :                             +- ShuffleQueryStage (24), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                +- Exchange (23)\n         :                                   +- * HashAggregate (22)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- BroadcastQueryStage (42)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (22)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]\n\n(29) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1344]\n\n(32) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]\n\n(42) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(43) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2367,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1344]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2414,"metricType":"sum"},{"name":"partition data size","accumulatorId":2415,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2416,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2410,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2411,"metricType":"timing"},{"name":"peak memory","accumulatorId":2409,"metricType":"size"},{"name":"number of output rows","accumulatorId":2408,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2413,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2412,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2404,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2405,"metricType":"timing"},{"name":"peak memory","accumulatorId":2403,"metricType":"size"},{"name":"number of output rows","accumulatorId":2402,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2407,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2406,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2401,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2399,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2393,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2400,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2394,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2391,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2388,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2397,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2390,"metricType":"sum"},{"name":"records read","accumulatorId":2387,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2385,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2386,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2383,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2389,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2382,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2392,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2379,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2395,"metricType":"size"},{"name":"number of partitions","accumulatorId":2380,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2396,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2384,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2398,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2375,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2376,"metricType":"timing"},{"name":"peak memory","accumulatorId":2374,"metricType":"size"},{"name":"number of output rows","accumulatorId":2373,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2378,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2377,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2372,"metricType":"timing"},{"name":"time to build","accumulatorId":2371,"metricType":"timing"},{"name":"time to collect","accumulatorId":2370,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2369,"metricType":"sum"},{"name":"data size","accumulatorId":2368,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2366,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2342,"metricType":"sum"},{"name":"partition data size","accumulatorId":2343,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2338,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2339,"metricType":"timing"},{"name":"peak memory","accumulatorId":2337,"metricType":"size"},{"name":"number of output rows","accumulatorId":2336,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2341,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2340,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2335,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2334,"metricType":"timing"},{"name":"time to build","accumulatorId":2333,"metricType":"timing"},{"name":"time to collect","accumulatorId":2332,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2331,"metricType":"sum"},{"name":"data size","accumulatorId":2330,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2365,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2363,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2357,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2364,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2358,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2355,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2352,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2361,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2354,"metricType":"sum"},{"name":"records read","accumulatorId":2351,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2349,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2350,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2347,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2353,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2346,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2356,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2345,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2359,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2360,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2348,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2362,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":18,"Submission Time":1784265096092,"Stage Infos":[{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"153\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[30,31,29],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 0d0cd66c-cfe0-4b96-a367-24dceaa51a70)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"153\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096096,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 0d0cd66c-cfe0-4b96-a367-24dceaa51a70)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":19,"Submission Time":1784265096116,"Stage Infos":[{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"149\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[33,32],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"149\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 116f3ae7-8332-4e37-88e7-885b46ac6096)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"149\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096118,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"149\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 116f3ae7-8332-4e37-88e7-885b46ac6096)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":31,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096116,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096132,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2414,1],[2416,1],[2415,4511]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2380,200]]}
{"Event":"SparkListenerJobStart","Job ID":20,"Submission Time":1784265096168,"Stage Infos":[{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[34,35],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096177,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096194,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096132,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096204,"Failed":false,"Killed":false,"Accumulables":[{"ID":1980,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1983,"Name":"local bytes read","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1985,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2335,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2336,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2337,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2339,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2340,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2452,"Name":"internal.metrics.executorDeserializeTime","Update":27,"Value":27,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9279356,"Value":9279356,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.executorRunTime","Update":28,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.executorCpuTime","Update":23304862,"Value":23304862,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.resultSize","Update":4939,"Value":4939,"Internal":true,"Count Failed Values":true},{"ID":2461,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2463,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2464,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2465,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2466,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2467,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2472,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2473,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2474,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2475,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2477,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":27,"Executor Deserialize CPU Time":9279356,"Executor Run Time":28,"Executor CPU Time":23304862,"Peak Execution Memory":4456432,"Result Size":4939,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":370,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"149\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096118,"Completion Time":1784265096206,"Accumulables":[{"ID":1980,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1983,"Name":"local bytes read","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1985,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2335,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2336,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2337,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2339,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2340,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2452,"Name":"internal.metrics.executorDeserializeTime","Value":27,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9279356,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.executorRunTime","Value":28,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.executorCpuTime","Value":23304862,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.resultSize","Value":4939,"Internal":true,"Count Failed Values":true},{"ID":2461,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2463,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2464,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2465,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2466,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2467,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":370,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2472,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2473,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2474,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2475,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2477,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":19,"Completion Time":1784265096206,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2334,5],[2333,5],[2332,145],[2331,5],[2330,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":31,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096116,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096265,"Failed":false,"Killed":false,"Accumulables":[{"ID":1830,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1833,"Name":"local bytes read","Update":"1849","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1834,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1835,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1859,"Name":"local bytes read","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1860,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"duration","Update":"25","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"duration","Update":"85","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2259,"Name":"sort time","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2260,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2261,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2265,"Name":"duration","Update":"48","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2266,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2267,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2268,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2417,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9339055,"Value":9339055,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.executorRunTime","Update":111,"Value":111,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.executorCpuTime","Update":54120940,"Value":54120940,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.resultSize","Update":6856,"Value":6856,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.jvmGCTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2426,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":2428,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2429,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2430,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2431,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2432,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3499,"Value":3499,"Internal":true,"Count Failed Values":true},{"ID":2433,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2434,"Name":"internal.metrics.shuffle.read.recordsRead","Update":65,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":2435,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2436,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2437,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2438,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2439,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2440,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2441,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2442,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2443,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2444,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":9339055,"Executor Run Time":111,"Executor CPU Time":54120940,"Peak Execution Memory":8519648,"Result Size":6856,"JVM GC Time":19,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3499,"Total Records Read":65,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"153\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096096,"Completion Time":1784265096266,"Accumulables":[{"ID":1830,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1833,"Name":"local bytes read","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1834,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1835,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1859,"Name":"local bytes read","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1860,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"duration","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"duration","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2259,"Name":"sort time","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2260,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2261,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2265,"Name":"duration","Value":"48","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2266,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2267,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2268,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2417,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":2418,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9339055,"Internal":true,"Count Failed Values":true},{"ID":2419,"Name":"internal.metrics.executorRunTime","Value":111,"Internal":true,"Count Failed Values":true},{"ID":2420,"Name":"internal.metrics.executorCpuTime","Value":54120940,"Internal":true,"Count Failed Values":true},{"ID":2421,"Name":"internal.metrics.resultSize","Value":6856,"Internal":true,"Count Failed Values":true},{"ID":2422,"Name":"internal.metrics.jvmGCTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":2426,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":2428,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2429,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2430,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2431,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2432,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3499,"Internal":true,"Count Failed Values":true},{"ID":2433,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2434,"Name":"internal.metrics.shuffle.read.recordsRead","Value":65,"Internal":true,"Count Failed Values":true},{"ID":2435,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2436,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2437,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2438,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2439,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2440,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2441,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2442,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2443,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2444,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":18,"Completion Time":1784265096266,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2254,2],[2253,3],[2252,262],[2251,45],[2250,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096194,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096318,"Failed":false,"Killed":false,"Accumulables":[{"ID":1948,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"local bytes read","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1952,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1953,"Name":"records read","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2379,"Name":"data size","Update":"1536","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2398,"Name":"shuffle bytes written","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2399,"Name":"shuffle records written","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2400,"Name":"shuffle write time","Update":"3188470","Value":"3188470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2401,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2402,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2403,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2405,"Name":"time in aggregation build","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2408,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2409,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2411,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"internal.metrics.executorDeserializeTime","Update":31,"Value":31,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8950941,"Value":8950941,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.executorRunTime","Update":82,"Value":82,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.executorCpuTime","Update":50003916,"Value":50003916,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.resultSize","Update":5603,"Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.jvmGCTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":2496,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2500,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2501,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2502,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.recordsRead","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2513,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2514,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2515,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2516,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2517,"Name":"internal.metrics.shuffle.write.writeTime","Update":3188470,"Value":3188470,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":31,"Executor Deserialize CPU Time":8950941,"Executor Run Time":82,"Executor CPU Time":50003916,"Peak Execution Memory":524288,"Result Size":5603,"JVM GC Time":19,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4311,"Total Records Read":54,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1884,"Shuffle Write Time":3188470,"Shuffle Records Written":24},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096177,"Completion Time":1784265096319,"Accumulables":[{"ID":1948,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"local bytes read","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1952,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1953,"Name":"records read","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2379,"Name":"data size","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2398,"Name":"shuffle bytes written","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2399,"Name":"shuffle records written","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2400,"Name":"shuffle write time","Value":"3188470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2401,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2402,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2403,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2405,"Name":"time in aggregation build","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2408,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2409,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2411,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"internal.metrics.executorDeserializeTime","Value":31,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8950941,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.executorRunTime","Value":82,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.executorCpuTime","Value":50003916,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.resultSize","Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.jvmGCTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":2496,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2500,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2501,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2502,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.recordsRead","Value":54,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2513,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2514,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2515,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2516,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":24,"Internal":true,"Count Failed Values":true},{"ID":2517,"Name":"internal.metrics.shuffle.write.writeTime","Value":3188470,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":20,"Completion Time":1784265096319,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (80)\n+- == Current Plan ==\n   CollectLimit (47)\n   +- Project (46)\n      +- BroadcastHashJoin LeftOuter BuildRight (45)\n         :- LocalLimit (36)\n         :  +- Project (35)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (34)\n         :        :- LocalLimit (20)\n         :        :  +- Project (19)\n         :        :     +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :        :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :           +- Exchange (16)\n         :        :              +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastQueryStage (33)\n         :           +- BroadcastExchange (32)\n         :              +- * HashAggregate (31)\n         :                 +- AQEShuffleRead (30)\n         :                    +- ShuffleQueryStage (29), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                       +- Exchange (28)\n         :                          +- * HashAggregate (27)\n         :                             +- * HashAggregate (26)\n         :                                +- AQEShuffleRead (25)\n         :                                   +- ShuffleQueryStage (24), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                      +- Exchange (23)\n         :                                         +- * HashAggregate (22)\n         :                                            +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- BroadcastQueryStage (44), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (43)\n               +- * HashAggregate (42)\n                  +- AQEShuffleRead (41)\n                     +- ShuffleQueryStage (40), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (39)\n                           +- * HashAggregate (38)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (79)\n   +- Project (78)\n      +- SortMergeJoin LeftOuter (77)\n         :- Sort (73)\n         :  +- Exchange (72)\n         :     +- LocalLimit (71)\n         :        +- Project (70)\n         :           +- SortMergeJoin LeftOuter (69)\n         :              :- Sort (62)\n         :              :  +- Exchange (61)\n         :              :     +- LocalLimit (60)\n         :              :        +- Project (59)\n         :              :           +- SortMergeJoin Inner (58)\n         :              :              :- Sort (55)\n         :              :              :  +- Exchange (54)\n         :              :              :     +- Project (53)\n         :              :              :        +- SortMergeJoin Inner (52)\n         :              :              :           :- Sort (49)\n         :              :              :           :  +- Exchange (48)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (51)\n         :              :              :              +- Exchange (50)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (57)\n         :              :                 +- Exchange (56)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (68)\n         :                 +- HashAggregate (67)\n         :                    +- Exchange (66)\n         :                       +- HashAggregate (65)\n         :                          +- HashAggregate (64)\n         :                             +- Exchange (63)\n         :                                +- HashAggregate (22)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (76)\n            +- HashAggregate (75)\n               +- Exchange (74)\n                  +- HashAggregate (38)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]\n\n(29) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(30) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(31) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(32) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1480]\n\n(33) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(34) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(35) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(36) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(37) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(38) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(39) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(40) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(41) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(43) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]\n\n(44) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(45) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(46) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(47) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(48) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(49) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(50) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(51) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(52) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(53) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(54) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(55) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(56) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(57) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(58) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(59) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(60) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(61) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(62) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(63) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(64) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(66) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(67) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(68) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(69) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(70) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(71) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(72) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(73) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(74) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(75) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(76) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(77) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(78) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(79) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(80) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2367,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1480]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2414,"metricType":"sum"},{"name":"partition data size","accumulatorId":2415,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2416,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2410,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2411,"metricType":"timing"},{"name":"peak memory","accumulatorId":2409,"metricType":"size"},{"name":"number of output rows","accumulatorId":2408,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2413,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2412,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2404,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2405,"metricType":"timing"},{"name":"peak memory","accumulatorId":2403,"metricType":"size"},{"name":"number of output rows","accumulatorId":2402,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2407,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2406,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2401,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2399,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2393,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2400,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2394,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2391,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2388,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2397,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2390,"metricType":"sum"},{"name":"records read","accumulatorId":2387,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2385,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2386,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2383,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2389,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2382,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2392,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2379,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2395,"metricType":"size"},{"name":"number of partitions","accumulatorId":2380,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2396,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2384,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2398,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2556,"metricType":"sum"},{"name":"partition data size","accumulatorId":2557,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2558,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2552,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2553,"metricType":"timing"},{"name":"peak memory","accumulatorId":2551,"metricType":"size"},{"name":"number of output rows","accumulatorId":2550,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2555,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2554,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2549,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2548,"metricType":"timing"},{"name":"time to build","accumulatorId":2547,"metricType":"timing"},{"name":"time to collect","accumulatorId":2546,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2545,"metricType":"sum"},{"name":"data size","accumulatorId":2544,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2543,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2342,"metricType":"sum"},{"name":"partition data size","accumulatorId":2343,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2338,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2339,"metricType":"timing"},{"name":"peak memory","accumulatorId":2337,"metricType":"size"},{"name":"number of output rows","accumulatorId":2336,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2341,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2340,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2335,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2334,"metricType":"timing"},{"name":"time to build","accumulatorId":2333,"metricType":"timing"},{"name":"time to collect","accumulatorId":2332,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2331,"metricType":"sum"},{"name":"data size","accumulatorId":2330,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2542,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2540,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2534,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2541,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2535,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2532,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2529,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2538,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2531,"metricType":"sum"},{"name":"records read","accumulatorId":2528,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2526,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2527,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2524,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2530,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2523,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2533,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2522,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2536,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2537,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2525,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2539,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2556,1],[2558,1],[2557,1975]]}
{"Event":"SparkListenerJobStart","Job ID":21,"Submission Time":1784265096355,"Stage Infos":[{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":36,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[37,38,36],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId eb961e64-2c62-4f0a-8232-4ae365738eed)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096358,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId eb961e64-2c62-4f0a-8232-4ae365738eed)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":38,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096362,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":38,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096362,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096415,"Failed":false,"Killed":false,"Accumulables":[{"ID":2382,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"local bytes read","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2387,"Name":"records read","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2549,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2550,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2553,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2554,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2559,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":2560,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7287084,"Value":7287084,"Internal":true,"Count Failed Values":true},{"ID":2561,"Name":"internal.metrics.executorRunTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":2562,"Name":"internal.metrics.executorCpuTime","Update":18938541,"Value":18938541,"Internal":true,"Count Failed Values":true},{"ID":2563,"Name":"internal.metrics.resultSize","Update":7206,"Value":7206,"Internal":true,"Count Failed Values":true},{"ID":2568,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2570,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2571,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2572,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2573,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2574,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2575,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2576,"Name":"internal.metrics.shuffle.read.recordsRead","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2577,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2583,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2584,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2585,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2586,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":7287084,"Executor Run Time":34,"Executor CPU Time":18938541,"Peak Execution Memory":4456432,"Result Size":7206,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1884,"Total Records Read":24,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096358,"Completion Time":1784265096416,"Accumulables":[{"ID":2382,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2385,"Name":"local bytes read","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2386,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2387,"Name":"records read","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2549,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2550,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2553,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2554,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2559,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":2560,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7287084,"Internal":true,"Count Failed Values":true},{"ID":2561,"Name":"internal.metrics.executorRunTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":2562,"Name":"internal.metrics.executorCpuTime","Value":18938541,"Internal":true,"Count Failed Values":true},{"ID":2563,"Name":"internal.metrics.resultSize","Value":7206,"Internal":true,"Count Failed Values":true},{"ID":2568,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2570,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2571,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2572,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2573,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2574,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2575,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2576,"Name":"internal.metrics.shuffle.read.recordsRead","Value":24,"Internal":true,"Count Failed Values":true},{"ID":2577,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2583,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2584,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2585,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2586,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":21,"Completion Time":1784265096416,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2548,2],[2547,7],[2546,82],[2545,24],[2544,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (81)\n+- == Final Plan ==\n   CollectLimit (48)\n   +- * Project (47)\n      +- * BroadcastHashJoin LeftOuter BuildRight (46)\n         :- * LocalLimit (37)\n         :  +- * Project (36)\n         :     +- * BroadcastHashJoin LeftOuter BuildRight (35)\n         :        :- * LocalLimit (21)\n         :        :  +- * Project (20)\n         :        :     +- * BroadcastHashJoin Inner BuildLeft (19)\n         :        :        :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- AQEShuffleRead (18)\n         :        :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :              +- Exchange (16)\n         :        :                 +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastQueryStage (34), Statistics(sizeInBytes=2.0 MiB, rowCount=24)\n         :           +- BroadcastExchange (33)\n         :              +- * HashAggregate (32)\n         :                 +- AQEShuffleRead (31)\n         :                    +- ShuffleQueryStage (30), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                       +- Exchange (29)\n         :                          +- * HashAggregate (28)\n         :                             +- * HashAggregate (27)\n         :                                +- AQEShuffleRead (26)\n         :                                   +- ShuffleQueryStage (25), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                      +- Exchange (24)\n         :                                         +- * HashAggregate (23)\n         :                                            +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- BroadcastQueryStage (45), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (44)\n               +- * HashAggregate (43)\n                  +- AQEShuffleRead (42)\n                     +- ShuffleQueryStage (41), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (40)\n                           +- * HashAggregate (39)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (38)\n+- == Initial Plan ==\n   CollectLimit (80)\n   +- Project (79)\n      +- SortMergeJoin LeftOuter (78)\n         :- Sort (74)\n         :  +- Exchange (73)\n         :     +- LocalLimit (72)\n         :        +- Project (71)\n         :           +- SortMergeJoin LeftOuter (70)\n         :              :- Sort (63)\n         :              :  +- Exchange (62)\n         :              :     +- LocalLimit (61)\n         :              :        +- Project (60)\n         :              :           +- SortMergeJoin Inner (59)\n         :              :              :- Sort (56)\n         :              :              :  +- Exchange (55)\n         :              :              :     +- Project (54)\n         :              :              :        +- SortMergeJoin Inner (53)\n         :              :              :           :- Sort (50)\n         :              :              :           :  +- Exchange (49)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (52)\n         :              :              :              +- Exchange (51)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (58)\n         :              :                 +- Exchange (57)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (69)\n         :                 +- HashAggregate (68)\n         :                    +- Exchange (67)\n         :                       +- HashAggregate (66)\n         :                          +- HashAggregate (65)\n         :                             +- Exchange (64)\n         :                                +- HashAggregate (23)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (77)\n            +- HashAggregate (76)\n               +- Exchange (75)\n                  +- HashAggregate (39)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (38)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: local\n\n(19) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(20) Project [codegen id : 12]\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(21) LocalLimit [codegen id : 12]\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(22) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(23) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(25) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]\n\n(30) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(31) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(32) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(33) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1480]\n\n(34) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(35) BroadcastHashJoin [codegen id : 12]\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(36) Project [codegen id : 12]\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(37) LocalLimit [codegen id : 12]\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(38) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(39) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(40) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(41) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(42) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(43) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(44) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]\n\n(45) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(46) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(47) Project [codegen id : 12]\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(48) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(49) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(50) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(51) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(52) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(54) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(55) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(56) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(57) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(58) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(60) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(61) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(62) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(63) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(66) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(67) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(68) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(69) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(70) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(71) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(72) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(73) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(74) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(75) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(76) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(77) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(78) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(79) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(80) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(81) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2618,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2617,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1480]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1332]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2414,"metricType":"sum"},{"name":"partition data size","accumulatorId":2415,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2416,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2410,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2411,"metricType":"timing"},{"name":"peak memory","accumulatorId":2409,"metricType":"size"},{"name":"number of output rows","accumulatorId":2408,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2413,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2412,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2404,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2405,"metricType":"timing"},{"name":"peak memory","accumulatorId":2403,"metricType":"size"},{"name":"number of output rows","accumulatorId":2402,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2407,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2406,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2401,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2399,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2393,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2400,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2394,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2391,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2388,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2397,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2390,"metricType":"sum"},{"name":"records read","accumulatorId":2387,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2385,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2386,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2383,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2389,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2382,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2392,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2381,"metricType":"sum"},{"name":"data size","accumulatorId":2379,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2395,"metricType":"size"},{"name":"number of partitions","accumulatorId":2380,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2396,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2384,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2398,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2556,"metricType":"sum"},{"name":"partition data size","accumulatorId":2557,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2558,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2552,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2553,"metricType":"timing"},{"name":"peak memory","accumulatorId":2551,"metricType":"size"},{"name":"number of output rows","accumulatorId":2550,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2555,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2554,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2549,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2548,"metricType":"timing"},{"name":"time to build","accumulatorId":2547,"metricType":"timing"},{"name":"time to collect","accumulatorId":2546,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2545,"metricType":"sum"},{"name":"data size","accumulatorId":2544,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2616,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1272]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2342,"metricType":"sum"},{"name":"partition data size","accumulatorId":2343,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2338,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2339,"metricType":"timing"},{"name":"peak memory","accumulatorId":2337,"metricType":"size"},{"name":"number of output rows","accumulatorId":2336,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2341,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2340,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2335,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2334,"metricType":"timing"},{"name":"time to build","accumulatorId":2333,"metricType":"timing"},{"name":"time to collect","accumulatorId":2332,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2331,"metricType":"sum"},{"name":"data size","accumulatorId":2330,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2615,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2614,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2612,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2606,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2613,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2607,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2604,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2601,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2610,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2603,"metricType":"sum"},{"name":"records read","accumulatorId":2600,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2598,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2599,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2596,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2602,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2595,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2605,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2594,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2608,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2609,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2597,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2611,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2618,1]]}
{"Event":"SparkListenerJobStart","Job ID":22,"Submission Time":1784265096469,"Stage Infos":[{"Stage ID":39,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[39,40],"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096470,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"37165","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784265088616","spark.rdd.compress":"True","spark.app.submitTime":"1784265087912","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:37165/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/jars/org.ow2.asm_asm-9.7.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717091129-0090","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:37165/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:37165/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:37165/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:37165/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:37165/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:37165/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:37165/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:37165/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:37165/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:37165/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:37165/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:37165/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:37165/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:37165/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:37165/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:37165/files/com.google.errorprone_error_prone_annotations-2.36.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":40,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096474,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":40,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784265096474,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784265096541,"Failed":false,"Killed":false,"Accumulables":[{"ID":1882,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1885,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1886,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1887,"Name":"records read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"number of output rows","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2616,"Name":"number of output rows","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"number of output rows","Update":"29","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2619,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":2620,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5684897,"Value":5684897,"Internal":true,"Count Failed Values":true},{"ID":2621,"Name":"internal.metrics.executorRunTime","Update":48,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":2622,"Name":"internal.metrics.executorCpuTime","Update":32197426,"Value":32197426,"Internal":true,"Count Failed Values":true},{"ID":2623,"Name":"internal.metrics.resultSize","Update":4011,"Value":4011,"Internal":true,"Count Failed Values":true},{"ID":2628,"Name":"internal.metrics.peakExecutionMemory","Update":12585936,"Value":12585936,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":5684897,"Executor Run Time":48,"Executor CPU Time":32197426,"Peak Execution Memory":12585936,"Result Size":4011,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784265096470,"Completion Time":1784265096542,"Accumulables":[{"ID":1882,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1885,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1886,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1887,"Name":"records read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"number of output rows","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2616,"Name":"number of output rows","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"number of output rows","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2619,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":2620,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5684897,"Internal":true,"Count Failed Values":true},{"ID":2621,"Name":"internal.metrics.executorRunTime","Value":48,"Internal":true,"Count Failed Values":true},{"ID":2622,"Name":"internal.metrics.executorCpuTime","Value":32197426,"Internal":true,"Count Failed Values":true},{"ID":2623,"Name":"internal.metrics.resultSize","Value":4011,"Internal":true,"Count Failed Values":true},{"ID":2628,"Name":"internal.metrics.peakExecutionMemory","Value":12585936,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":22,"Completion Time":1784265096542,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1784265096552,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"rootExecutionId":3,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nExecute SaveIntoDataSourceCommand (1)\n   +- SaveIntoDataSourceCommand (2)\n         +- Project (12)\n            +- Aggregate (11)\n               +- Join (10)\n                  :- Join (7)\n                  :  :- SubqueryAlias (4)\n                  :  :  +- LogicalRelation (3)\n                  :  +- SubqueryAlias (6)\n                  :     +- LogicalRelation (5)\n                  +- SubqueryAlias (9)\n                     +- LogicalRelation (8)\n\n\n(1) Execute SaveIntoDataSourceCommand\nOutput: []\n\n(2) SaveIntoDataSourceCommand\nArguments: org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider@22ee661f, [url=*********(redacted), driver=com.clickhouse.jdbc.ClickHouseDriver, dbtable=dm_lesson_popularity_summary, user=anazarenko, password=*********(redacted)], Overwrite\n\n(3) LogicalRelation\nArguments: JDBCRelation(core_dim_lesson) [numPartitions=1], [lesson_id#18, title#19, duration_min#20L, course_id#21], false\n\n(4) SubqueryAlias\nArguments: l\n\n(5) LogicalRelation\nArguments: JDBCRelation(core_dim_course) [numPartitions=1], [course_id#10, title#11, category#12, created_at#13], false\n\n(6) SubqueryAlias\nArguments: c\n\n(7) Join\nArguments: Inner, (course_id#10 = course_id#21)\n\n(8) LogicalRelation\nArguments: JDBCRelation(core_fact_lesson_views) [numPartitions=1], [user_id#32, lesson_id#33, course_id#34, viewed_at#35], false\n\n(9) SubqueryAlias\nArguments: lv\n\n(10) Join\nArguments: LeftOuter, (lesson_id#18 = lesson_id#33)\n\n(11) Aggregate\nArguments: [lesson_id#18, title#19, title#11, course_id#21], [lesson_id#18, title#19, title#11, course_id#21, count(lesson_id#33) AS total_views#93L, count(distinct user_id#32) AS unique_users#94L, min(viewed_at#35) AS first_view#96, max(viewed_at#35) AS last_view#98]\n\n(12) Project\nArguments: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\n\n","sparkPlanInfo":{"nodeName":"Execute SaveIntoDataSourceCommand","simpleString":"Execute SaveIntoDataSourceCommand","children":[],"metadata":{},"metrics":[]},"time":1784265096578,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1784265096619,"errorMessage":"java.sql.SQLException: Code: 42. DB::Exception: ORDER BY or PRIMARY KEY clause is missing. Consider using extended storage definition syntax with ORDER BY or PRIMARY KEY clause. With deprecated old syntax (highly not recommended) storage MergeTree requires 3 to 4 parameters: \nname of column with date,\n[sampling element of primary key],\nprimary key expression,\nindex granularity\n\nSyntax for the MergeTree table engine:\n\nCREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]\n(\n    name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],\n    name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],\n    ...\n    INDEX index_name1 expr1 TYPE type1(...) [GRANULARITY value1],\n    INDEX index_name2 expr2 TYPE type2(...) [GRANULARITY value2]\n) ENGINE = MergeTree()\nORDER BY expr\n[PARTITION BY expr]\n[PRIMARY KEY expr]\n[SAMPLE BY expr]\n[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...]\n[SETTINGS name=value, ...]\n[COMMENT 'comment']\n\nSee details in documentation: https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree/. Other engines of the family support different syntax, see details in the corresponding documentation topics.\n\nIf you use the Replicated version of engines, see https://clickhouse.com/docs/engines/table-engines/mergetree-family/replication/.\n. (NUMBER_OF_ARGUMENTS_DOESNT_MATCH) (version 25.9.3.48 (official build))\n, server ClickHouseNode [uri=http://clickhouse.de-infra.servehttp.com:9004/anazarenko]@890499824\n\tat com.clickhouse.jdbc.SqlExceptionUtils.handle(SqlExceptionUtils.java:85)\n\tat com.clickhouse.jdbc.SqlExceptionUtils.create(SqlExceptionUtils.java:31)\n\tat com.clickhouse.jdbc.SqlExceptionUtils.handle(SqlExceptionUtils.java:90)\n\tat com.clickhouse.jdbc.internal.ClickHouseStatementImpl.getLastResponse(ClickHouseStatementImpl.java:122)\n\tat com.clickhouse.jdbc.internal.ClickHouseStatementImpl.executeLargeUpdate(ClickHouseStatementImpl.java:487)\n\tat com.clickhouse.jdbc.internal.ClickHouseStatementImpl.executeUpdate(ClickHouseStatementImpl.java:496)\n\tat org.apache.spark.sql.jdbc.JdbcDialect.createTable(JdbcDialects.scala:191)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.createTable(JdbcUtils.scala:921)\n\tat org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:64)\n\tat org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:48)\n\tat org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)\n\tat org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)\n\tat org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84)\n\tat org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:107)\n\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:125)\n\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:201)\n\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:108)\n\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:900)\n\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:66)\n\tat org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:107)\n\tat org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:98)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:461)\n\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:76)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:461)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:32)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:32)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:437)\n\tat org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:98)\n\tat org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:85)\n\tat org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:83)\n\tat org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:142)\n\tat org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:859)\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:388)\n\tat org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:361)\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\n\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.base/java.lang.reflect.Method.invoke(Method.java:569)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\nCaused by: java.io.IOException: Code: 42. DB::Exception: ORDER BY or PRIMARY KEY clause is missing. Consider using extended storage definition syntax with ORDER BY or PRIMARY KEY clause. With deprecated old syntax (highly not recommended) storage MergeTree requires 3 to 4 parameters: \nname of column with date,\n[sampling element of primary key],\nprimary key expression,\nindex granularity\n\nSyntax for the MergeTree table engine:\n\nCREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]\n(\n    name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],\n    name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],\n    ...\n    INDEX index_name1 expr1 TYPE type1(...) [GRANULARITY value1],\n    INDEX index_name2 expr2 TYPE type2(...) [GRANULARITY value2]\n) ENGINE = MergeTree()\nORDER BY expr\n[PARTITION BY expr]\n[PRIMARY KEY expr]\n[SAMPLE BY expr]\n[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...]\n[SETTINGS name=value, ...]\n[COMMENT 'comment']\n\nSee details in documentation: https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree/. Other engines of the family support different syntax, see details in the corresponding documentation topics.\n\nIf you use the Replicated version of engines, see https://clickhouse.com/docs/engines/table-engines/mergetree-family/replication/.\n. (NUMBER_OF_ARGUMENTS_DOESNT_MATCH) (version 25.9.3.48 (official build))\n\n\tat com.clickhouse.client.http.ApacheHttpConnectionImpl.checkResponse(ApacheHttpConnectionImpl.java:209)\n\tat com.clickhouse.client.http.ApacheHttpConnectionImpl.post(ApacheHttpConnectionImpl.java:243)\n\tat com.clickhouse.client.http.ClickHouseHttpClient.send(ClickHouseHttpClient.java:118)\n\tat com.clickhouse.client.AbstractClient.execute(AbstractClient.java:280)\n\tat com.clickhouse.client.ClickHouseClientBuilder$Agent.sendOnce(ClickHouseClientBuilder.java:282)\n\tat com.clickhouse.client.ClickHouseClientBuilder$Agent.send(ClickHouseClientBuilder.java:294)\n\tat com.clickhouse.client.ClickHouseClientBuilder$Agent.execute(ClickHouseClientBuilder.java:349)\n\tat com.clickhouse.client.ClickHouseClient.executeAndWait(ClickHouseClient.java:878)\n\tat com.clickhouse.client.ClickHouseRequest.executeAndWait(ClickHouseRequest.java:2154)\n\tat com.clickhouse.jdbc.internal.ClickHouseStatementImpl.getLastResponse(ClickHouseStatementImpl.java:120)\n\t... 46 more\n"}
{"Event":"SparkListenerApplicationEnd","Timestamp":1784265096792}